2024-12-03 02:33:13,843 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 02:33:13,865 main DEBUG Took 0.019177 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 02:33:13,866 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 02:33:13,866 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 02:33:13,867 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 02:33:13,875 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,888 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 02:33:13,907 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,909 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,909 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,911 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,912 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,912 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,913 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,914 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,915 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,915 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,916 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,917 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,917 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,917 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,918 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,919 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:33:13,922 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,923 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 02:33:13,925 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:33:13,926 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 02:33:13,929 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 02:33:13,930 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 02:33:13,931 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 02:33:13,932 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 02:33:13,941 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 02:33:13,944 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 02:33:13,947 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 02:33:13,947 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 02:33:13,949 main DEBUG createAppenders(={Console}) 2024-12-03 02:33:13,950 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-03 02:33:13,951 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 02:33:13,951 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-03 02:33:13,952 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 02:33:13,952 main DEBUG OutputStream closed 2024-12-03 02:33:13,952 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 02:33:13,953 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 02:33:13,953 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-03 02:33:14,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 02:33:14,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 02:33:14,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 02:33:14,076 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 02:33:14,076 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 02:33:14,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 02:33:14,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 02:33:14,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 02:33:14,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 02:33:14,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 02:33:14,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 02:33:14,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 02:33:14,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 02:33:14,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 02:33:14,080 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 02:33:14,080 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 02:33:14,080 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 02:33:14,081 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 02:33:14,083 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 02:33:14,089 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-03 02:33:14,089 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 02:33:14,092 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-03T02:33:14,509 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e 2024-12-03 02:33:14,513 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 02:33:14,514 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T02:33:14,525 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-03T02:33:14,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T02:33:14,596 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d, deleteOnExit=true 2024-12-03T02:33:14,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T02:33:14,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/test.cache.data in system properties and HBase conf 2024-12-03T02:33:14,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T02:33:14,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/hadoop.log.dir in system properties and HBase conf 2024-12-03T02:33:14,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T02:33:14,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T02:33:14,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T02:33:14,736 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T02:33:14,858 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T02:33:14,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T02:33:14,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T02:33:14,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T02:33:14,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T02:33:14,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T02:33:14,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T02:33:14,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T02:33:14,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T02:33:14,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T02:33:14,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/nfs.dump.dir in system properties and HBase conf 2024-12-03T02:33:14,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/java.io.tmpdir in system properties and HBase conf 2024-12-03T02:33:14,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T02:33:14,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T02:33:14,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T02:33:16,197 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T02:33:16,272 INFO [Time-limited test {}] log.Log(170): Logging initialized @3557ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T02:33:16,349 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:33:16,415 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T02:33:16,439 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T02:33:16,439 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T02:33:16,441 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T02:33:16,458 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:33:16,462 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/hadoop.log.dir/,AVAILABLE} 2024-12-03T02:33:16,463 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T02:33:16,729 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/java.io.tmpdir/jetty-localhost-41459-hadoop-hdfs-3_4_1-tests_jar-_-any-15369585755909033858/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T02:33:16,736 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41459} 2024-12-03T02:33:16,737 INFO [Time-limited test {}] server.Server(415): Started @4022ms 2024-12-03T02:33:17,783 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:33:17,826 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T02:33:17,880 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T02:33:17,881 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T02:33:17,882 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T02:33:17,889 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/hadoop.log.dir/,AVAILABLE} 2024-12-03T02:33:17,890 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T02:33:18,099 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/java.io.tmpdir/jetty-localhost-42807-hadoop-hdfs-3_4_1-tests_jar-_-any-1110516091809859417/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:33:18,100 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:42807} 2024-12-03T02:33:18,100 INFO [Time-limited test {}] server.Server(415): Started @5386ms 2024-12-03T02:33:18,167 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T02:33:19,826 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d/data/data1/current/BP-660169850-172.17.0.3-1733193195623/current, will proceed with Du for space computation calculation, 2024-12-03T02:33:19,832 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d/data/data2/current/BP-660169850-172.17.0.3-1733193195623/current, will proceed with Du for space computation calculation, 2024-12-03T02:33:19,907 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T02:33:19,963 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9b916f1d5bfac415 with lease ID 0x9acf380c88adba8e: Processing first storage report for DS-47d93547-a507-44da-824f-e23e7bc84727 from datanode DatanodeRegistration(127.0.0.1:45879, datanodeUuid=b4b48bdb-1fce-440a-b941-c4b6a491ed21, infoPort=45403, infoSecurePort=0, ipcPort=45755, storageInfo=lv=-57;cid=testClusterID;nsid=1571835365;c=1733193195623) 2024-12-03T02:33:19,965 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9b916f1d5bfac415 with lease ID 0x9acf380c88adba8e: from storage DS-47d93547-a507-44da-824f-e23e7bc84727 node DatanodeRegistration(127.0.0.1:45879, datanodeUuid=b4b48bdb-1fce-440a-b941-c4b6a491ed21, infoPort=45403, infoSecurePort=0, ipcPort=45755, storageInfo=lv=-57;cid=testClusterID;nsid=1571835365;c=1733193195623), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-03T02:33:19,966 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9b916f1d5bfac415 with lease ID 0x9acf380c88adba8e: Processing first storage report for DS-401b6cac-c480-4764-8c26-290a0971bd4a from datanode DatanodeRegistration(127.0.0.1:45879, datanodeUuid=b4b48bdb-1fce-440a-b941-c4b6a491ed21, infoPort=45403, infoSecurePort=0, ipcPort=45755, storageInfo=lv=-57;cid=testClusterID;nsid=1571835365;c=1733193195623) 2024-12-03T02:33:19,967 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9b916f1d5bfac415 with lease ID 0x9acf380c88adba8e: from storage DS-401b6cac-c480-4764-8c26-290a0971bd4a node DatanodeRegistration(127.0.0.1:45879, datanodeUuid=b4b48bdb-1fce-440a-b941-c4b6a491ed21, infoPort=45403, infoSecurePort=0, ipcPort=45755, storageInfo=lv=-57;cid=testClusterID;nsid=1571835365;c=1733193195623), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T02:33:19,996 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e 2024-12-03T02:33:20,157 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d/zookeeper_0, clientPort=60541, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T02:33:20,173 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60541 2024-12-03T02:33:20,187 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:33:20,191 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:33:20,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741825_1001 (size=7) 2024-12-03T02:33:20,911 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37 with version=8 2024-12-03T02:33:20,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/hbase-staging 2024-12-03T02:33:21,009 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T02:33:21,332 INFO [Time-limited test {}] client.ConnectionUtils(128): master/64035cbb0b46:0 server-side Connection retries=45 2024-12-03T02:33:21,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:33:21,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T02:33:21,359 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T02:33:21,359 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:33:21,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T02:33:21,600 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T02:33:21,694 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T02:33:21,721 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T02:33:21,728 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T02:33:21,773 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 88828 (auto-detected) 2024-12-03T02:33:21,775 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T02:33:21,806 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44739 2024-12-03T02:33:21,850 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44739 connecting to ZooKeeper ensemble=127.0.0.1:60541 2024-12-03T02:33:21,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:447390x0, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T02:33:21,971 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44739-0x1019a5323e20000 connected 2024-12-03T02:33:22,159 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:33:22,162 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:33:22,173 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:33:22,178 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37, hbase.cluster.distributed=false 2024-12-03T02:33:22,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T02:33:22,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44739 2024-12-03T02:33:22,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44739 2024-12-03T02:33:22,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44739 2024-12-03T02:33:22,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44739 2024-12-03T02:33:22,236 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44739 2024-12-03T02:33:22,389 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/64035cbb0b46:0 server-side Connection retries=45 2024-12-03T02:33:22,390 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:33:22,391 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T02:33:22,391 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T02:33:22,391 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:33:22,391 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T02:33:22,394 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T02:33:22,397 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T02:33:22,398 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38721 2024-12-03T02:33:22,400 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38721 connecting to ZooKeeper ensemble=127.0.0.1:60541 2024-12-03T02:33:22,401 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:33:22,407 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:33:22,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387210x0, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T02:33:22,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387210x0, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:33:22,434 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T02:33:22,439 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38721-0x1019a5323e20001 connected 2024-12-03T02:33:22,460 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T02:33:22,463 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T02:33:22,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T02:33:22,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38721 2024-12-03T02:33:22,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38721 2024-12-03T02:33:22,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38721 2024-12-03T02:33:22,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38721 2024-12-03T02:33:22,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38721 2024-12-03T02:33:22,523 DEBUG [M:0;64035cbb0b46:44739 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;64035cbb0b46:44739 2024-12-03T02:33:22,524 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/64035cbb0b46,44739,1733193201066 2024-12-03T02:33:22,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:33:22,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:33:22,553 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/64035cbb0b46,44739,1733193201066 2024-12-03T02:33:22,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T02:33:22,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:22,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:22,610 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T02:33:22,612 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/64035cbb0b46,44739,1733193201066 from backup master directory 2024-12-03T02:33:22,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:33:22,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/64035cbb0b46,44739,1733193201066 2024-12-03T02:33:22,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:33:22,632 WARN [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T02:33:22,632 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=64035cbb0b46,44739,1733193201066 2024-12-03T02:33:22,641 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T02:33:22,649 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T02:33:22,750 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/hbase.id] with ID: be4e62ab-a60c-4866-94e8-6b4f3144907d 2024-12-03T02:33:22,750 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/.tmp/hbase.id 2024-12-03T02:33:22,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741826_1002 (size=42) 2024-12-03T02:33:22,764 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/.tmp/hbase.id]:[hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/hbase.id] 2024-12-03T02:33:22,830 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:33:22,842 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T02:33:22,869 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-12-03T02:33:22,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:22,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:22,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741827_1003 (size=196) 2024-12-03T02:33:22,916 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:33:22,918 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T02:33:22,934 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:22,938 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:33:22,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741828_1004 (size=1189) 2024-12-03T02:33:23,392 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store 2024-12-03T02:33:23,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741829_1005 (size=34) 2024-12-03T02:33:23,815 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T02:33:23,817 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:33:23,818 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T02:33:23,819 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:33:23,819 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:33:23,821 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T02:33:23,821 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:33:23,821 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:33:23,823 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733193203818Disabling compacts and flushes for region at 1733193203818Disabling writes for close at 1733193203821 (+3 ms)Writing region close event to WAL at 1733193203821Closed at 1733193203821 2024-12-03T02:33:23,825 WARN [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/.initializing 2024-12-03T02:33:23,825 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/WALs/64035cbb0b46,44739,1733193201066 2024-12-03T02:33:23,835 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:33:23,852 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C44739%2C1733193201066, suffix=, logDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/WALs/64035cbb0b46,44739,1733193201066, archiveDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/oldWALs, maxLogs=10 2024-12-03T02:33:23,883 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/WALs/64035cbb0b46,44739,1733193201066/64035cbb0b46%2C44739%2C1733193201066.1733193203858, exclude list is [], retry=0 2024-12-03T02:33:23,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45879,DS-47d93547-a507-44da-824f-e23e7bc84727,DISK] 2024-12-03T02:33:23,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T02:33:23,940 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/WALs/64035cbb0b46,44739,1733193201066/64035cbb0b46%2C44739%2C1733193201066.1733193203858 2024-12-03T02:33:23,941 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45403:45403)] 2024-12-03T02:33:23,941 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:33:23,942 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:33:23,945 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:23,946 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:23,989 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,021 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T02:33:24,026 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:24,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:33:24,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,037 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T02:33:24,037 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:24,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:24,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,045 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T02:33:24,045 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:24,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:24,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T02:33:24,057 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:24,064 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:24,065 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,070 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,072 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,081 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,082 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,094 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T02:33:24,100 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:33:24,115 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:33:24,123 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60273155, jitterRate=-0.10186000168323517}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T02:33:24,131 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733193203957Initializing all the Stores at 1733193203960 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733193203961 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193203961Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193203962 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193203962Cleaning up temporary data from old regions at 1733193204082 (+120 ms)Region opened successfully at 1733193204131 (+49 ms) 2024-12-03T02:33:24,134 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T02:33:24,191 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f84067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=64035cbb0b46/172.17.0.3:0 2024-12-03T02:33:24,233 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T02:33:24,250 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T02:33:24,250 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T02:33:24,255 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T02:33:24,264 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 8 msec 2024-12-03T02:33:24,271 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-03T02:33:24,272 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T02:33:24,353 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T02:33:24,368 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T02:33:24,374 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T02:33:24,378 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T02:33:24,383 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T02:33:24,398 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T02:33:24,403 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T02:33:24,414 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T02:33:24,424 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T02:33:24,427 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T02:33:24,432 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T02:33:24,455 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T02:33:24,465 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T02:33:24,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T02:33:24,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T02:33:24,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:24,476 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=64035cbb0b46,44739,1733193201066, sessionid=0x1019a5323e20000, setting cluster-up flag (Was=false) 2024-12-03T02:33:24,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:24,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:24,550 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T02:33:24,552 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=64035cbb0b46,44739,1733193201066 2024-12-03T02:33:24,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:24,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:24,598 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T02:33:24,601 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=64035cbb0b46,44739,1733193201066 2024-12-03T02:33:24,608 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T02:33:24,687 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T02:33:24,696 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T02:33:24,702 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T02:33:24,710 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(746): ClusterId : be4e62ab-a60c-4866-94e8-6b4f3144907d 2024-12-03T02:33:24,708 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 64035cbb0b46,44739,1733193201066 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T02:33:24,713 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T02:33:24,717 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:33:24,717 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:33:24,717 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:33:24,718 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:33:24,718 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/64035cbb0b46:0, corePoolSize=10, maxPoolSize=10 2024-12-03T02:33:24,718 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,718 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/64035cbb0b46:0, corePoolSize=2, maxPoolSize=2 2024-12-03T02:33:24,718 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,722 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733193234722 2024-12-03T02:33:24,724 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T02:33:24,725 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T02:33:24,726 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T02:33:24,726 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T02:33:24,727 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T02:33:24,728 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T02:33:24,729 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T02:33:24,730 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T02:33:24,730 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T02:33:24,730 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T02:33:24,733 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T02:33:24,734 DEBUG [RS:0;64035cbb0b46:38721 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f509a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=64035cbb0b46/172.17.0.3:0 2024-12-03T02:33:24,732 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,735 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:24,735 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T02:33:24,740 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T02:33:24,741 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T02:33:24,741 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T02:33:24,743 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T02:33:24,744 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T02:33:24,751 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;64035cbb0b46:38721 2024-12-03T02:33:24,751 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.large.0-1733193204745,5,FailOnTimeoutGroup] 2024-12-03T02:33:24,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741831_1007 (size=1321) 2024-12-03T02:33:24,755 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.small.0-1733193204752,5,FailOnTimeoutGroup] 2024-12-03T02:33:24,755 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,756 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T02:33:24,756 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T02:33:24,756 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T02:33:24,756 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T02:33:24,757 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,758 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,759 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(2659): reportForDuty to master=64035cbb0b46,44739,1733193201066 with port=38721, startcode=1733193202350 2024-12-03T02:33:24,773 DEBUG [RS:0;64035cbb0b46:38721 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T02:33:24,806 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46003, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T02:33:24,812 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44739 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 64035cbb0b46,38721,1733193202350 2024-12-03T02:33:24,815 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44739 {}] master.ServerManager(517): Registering regionserver=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:24,828 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37 2024-12-03T02:33:24,828 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39485 2024-12-03T02:33:24,828 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T02:33:24,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T02:33:24,849 DEBUG [RS:0;64035cbb0b46:38721 {}] zookeeper.ZKUtil(111): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/64035cbb0b46,38721,1733193202350 2024-12-03T02:33:24,849 WARN [RS:0;64035cbb0b46:38721 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T02:33:24,850 INFO [RS:0;64035cbb0b46:38721 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:33:24,850 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/WALs/64035cbb0b46,38721,1733193202350 2024-12-03T02:33:24,852 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [64035cbb0b46,38721,1733193202350] 2024-12-03T02:33:24,875 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T02:33:24,886 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T02:33:24,891 INFO [RS:0;64035cbb0b46:38721 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T02:33:24,891 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,892 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T02:33:24,897 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T02:33:24,898 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,898 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,899 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,899 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,899 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,899 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,899 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0, corePoolSize=2, maxPoolSize=2 2024-12-03T02:33:24,899 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,900 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,900 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,900 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,900 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,900 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:33:24,900 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:33:24,901 DEBUG [RS:0;64035cbb0b46:38721 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:33:24,901 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,902 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,902 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,902 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,902 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,902 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,38721,1733193202350-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T02:33:24,919 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T02:33:24,920 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,38721,1733193202350-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,920 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,921 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.Replication(171): 64035cbb0b46,38721,1733193202350 started 2024-12-03T02:33:24,936 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:24,936 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1482): Serving as 64035cbb0b46,38721,1733193202350, RpcServer on 64035cbb0b46/172.17.0.3:38721, sessionid=0x1019a5323e20001 2024-12-03T02:33:24,936 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T02:33:24,937 DEBUG [RS:0;64035cbb0b46:38721 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 64035cbb0b46,38721,1733193202350 2024-12-03T02:33:24,937 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,38721,1733193202350' 2024-12-03T02:33:24,937 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T02:33:24,938 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T02:33:24,938 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T02:33:24,938 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T02:33:24,939 DEBUG [RS:0;64035cbb0b46:38721 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 64035cbb0b46,38721,1733193202350 2024-12-03T02:33:24,939 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,38721,1733193202350' 2024-12-03T02:33:24,939 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T02:33:24,939 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T02:33:24,940 DEBUG [RS:0;64035cbb0b46:38721 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T02:33:24,940 INFO [RS:0;64035cbb0b46:38721 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T02:33:24,940 INFO [RS:0;64035cbb0b46:38721 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T02:33:25,045 INFO [RS:0;64035cbb0b46:38721 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:33:25,048 INFO [RS:0;64035cbb0b46:38721 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C38721%2C1733193202350, suffix=, logDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/WALs/64035cbb0b46,38721,1733193202350, archiveDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/oldWALs, maxLogs=32 2024-12-03T02:33:25,068 DEBUG [RS:0;64035cbb0b46:38721 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/WALs/64035cbb0b46,38721,1733193202350/64035cbb0b46%2C38721%2C1733193202350.1733193205051, exclude list is [], retry=0 2024-12-03T02:33:25,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45879,DS-47d93547-a507-44da-824f-e23e7bc84727,DISK] 2024-12-03T02:33:25,078 INFO [RS:0;64035cbb0b46:38721 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/WALs/64035cbb0b46,38721,1733193202350/64035cbb0b46%2C38721%2C1733193202350.1733193205051 2024-12-03T02:33:25,079 DEBUG [RS:0;64035cbb0b46:38721 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45403:45403)] 2024-12-03T02:33:25,156 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T02:33:25,157 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37 2024-12-03T02:33:25,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741833_1009 (size=32) 2024-12-03T02:33:25,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:33:25,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T02:33:25,579 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T02:33:25,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:25,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:33:25,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T02:33:25,588 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T02:33:25,588 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:25,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:33:25,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T02:33:25,594 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T02:33:25,594 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:25,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:33:25,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T02:33:25,600 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T02:33:25,600 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:25,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:33:25,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T02:33:25,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740 2024-12-03T02:33:25,610 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740 2024-12-03T02:33:25,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T02:33:25,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T02:33:25,617 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:33:25,621 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T02:33:25,631 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:33:25,632 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62115783, jitterRate=-0.07440270483493805}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:33:25,635 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733193205572Initializing all the Stores at 1733193205574 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733193205574Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733193205575 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193205575Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733193205575Cleaning up temporary data from old regions at 1733193205615 (+40 ms)Region opened successfully at 1733193205635 (+20 ms) 2024-12-03T02:33:25,635 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T02:33:25,636 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T02:33:25,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T02:33:25,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T02:33:25,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T02:33:25,644 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T02:33:25,644 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733193205635Disabling compacts and flushes for region at 1733193205635Disabling writes for close at 1733193205636 (+1 ms)Writing region close event to WAL at 1733193205640 (+4 ms)Closed at 1733193205643 (+3 ms) 2024-12-03T02:33:25,653 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T02:33:25,653 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T02:33:25,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T02:33:25,667 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T02:33:25,670 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T02:33:25,823 DEBUG [64035cbb0b46:44739 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T02:33:25,836 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:25,843 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 64035cbb0b46,38721,1733193202350, state=OPENING 2024-12-03T02:33:25,873 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T02:33:25,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:25,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:33:25,949 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:33:25,949 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:33:25,951 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T02:33:25,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:33:26,134 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T02:33:26,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38783, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T02:33:26,172 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T02:33:26,172 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:33:26,173 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T02:33:26,181 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C38721%2C1733193202350.meta, suffix=.meta, logDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/WALs/64035cbb0b46,38721,1733193202350, archiveDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/oldWALs, maxLogs=32 2024-12-03T02:33:26,198 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/WALs/64035cbb0b46,38721,1733193202350/64035cbb0b46%2C38721%2C1733193202350.meta.1733193206183.meta, exclude list is [], retry=0 2024-12-03T02:33:26,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45879,DS-47d93547-a507-44da-824f-e23e7bc84727,DISK] 2024-12-03T02:33:26,207 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/WALs/64035cbb0b46,38721,1733193202350/64035cbb0b46%2C38721%2C1733193202350.meta.1733193206183.meta 2024-12-03T02:33:26,210 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45403:45403)] 2024-12-03T02:33:26,210 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:33:26,212 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T02:33:26,215 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T02:33:26,220 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T02:33:26,225 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T02:33:26,226 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:33:26,226 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T02:33:26,226 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T02:33:26,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T02:33:26,234 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T02:33:26,234 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:26,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:33:26,236 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T02:33:26,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T02:33:26,237 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:26,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:33:26,239 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T02:33:26,241 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T02:33:26,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:26,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:33:26,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T02:33:26,246 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T02:33:26,246 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:26,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:33:26,247 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T02:33:26,250 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740 2024-12-03T02:33:26,253 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740 2024-12-03T02:33:26,257 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T02:33:26,257 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T02:33:26,259 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:33:26,263 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T02:33:26,265 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61857047, jitterRate=-0.07825817167758942}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:33:26,266 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T02:33:26,267 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733193206227Writing region info on filesystem at 1733193206228 (+1 ms)Initializing all the Stores at 1733193206231 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733193206231Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733193206231Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193206231Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733193206231Cleaning up temporary data from old regions at 1733193206257 (+26 ms)Running coprocessor post-open hooks at 1733193206266 (+9 ms)Region opened successfully at 1733193206267 (+1 ms) 2024-12-03T02:33:26,274 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733193206122 2024-12-03T02:33:26,285 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T02:33:26,286 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T02:33:26,289 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:26,292 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 64035cbb0b46,38721,1733193202350, state=OPEN 2024-12-03T02:33:26,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T02:33:26,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T02:33:26,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:33:26,374 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:33:26,375 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:26,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T02:33:26,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=64035cbb0b46,38721,1733193202350 in 422 msec 2024-12-03T02:33:26,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T02:33:26,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 725 msec 2024-12-03T02:33:26,392 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T02:33:26,392 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T02:33:26,417 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:26,419 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:26,461 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:26,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55141, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:26,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.8530 sec 2024-12-03T02:33:26,493 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733193206493, completionTime=-1 2024-12-03T02:33:26,496 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T02:33:26,497 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T02:33:26,525 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T02:33:26,525 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733193266525 2024-12-03T02:33:26,525 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733193326525 2024-12-03T02:33:26,526 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-03T02:33:26,528 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,44739,1733193201066-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:26,528 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,44739,1733193201066-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:26,528 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,44739,1733193201066-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:26,529 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-64035cbb0b46:44739, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:26,530 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:26,530 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:26,536 DEBUG [master/64035cbb0b46:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T02:33:26,555 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.923sec 2024-12-03T02:33:26,557 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T02:33:26,558 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T02:33:26,559 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T02:33:26,559 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T02:33:26,559 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T02:33:26,560 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,44739,1733193201066-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T02:33:26,561 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,44739,1733193201066-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T02:33:26,568 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T02:33:26,569 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T02:33:26,570 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,44739,1733193201066-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:33:26,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:26,624 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T02:33:26,624 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T02:33:26,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:26,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:26,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb5a2c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:26,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:26,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:26,661 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:26,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:26,668 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:26,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:26,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:26,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:26,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=64035cbb0b46,44739,1733193201066 2024-12-03T02:33:26,729 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=758, ProcessCount=11, AvailableMemoryMB=3273 2024-12-03T02:33:26,740 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T02:33:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 64035cbb0b46,44739,1733193201066 2024-12-03T02:33:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@453fbba3 2024-12-03T02:33:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T02:33:26,756 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T02:33:26,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T02:33:26,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:33:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:26,785 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T02:33:26,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-03T02:33:26,788 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:26,791 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T02:33:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:33:26,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741835_1011 (size=1061) 2024-12-03T02:33:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:33:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:33:27,217 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37 2024-12-03T02:33:27,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741836_1012 (size=53) 2024-12-03T02:33:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:33:27,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:33:27,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a2199fc0aeeef0203e54ad44ea3ac1a7, disabling compactions & flushes 2024-12-03T02:33:27,630 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:27,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:27,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. after waiting 0 ms 2024-12-03T02:33:27,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:27,630 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:27,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a2199fc0aeeef0203e54ad44ea3ac1a7: Waiting for close lock at 1733193207630Disabling compacts and flushes for region at 1733193207630Disabling writes for close at 1733193207630Writing region close event to WAL at 1733193207630Closed at 1733193207630 2024-12-03T02:33:27,633 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T02:33:27,637 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733193207633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733193207633"}]},"ts":"1733193207633"} 2024-12-03T02:33:27,643 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T02:33:27,645 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T02:33:27,648 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193207645"}]},"ts":"1733193207645"} 2024-12-03T02:33:27,652 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T02:33:27,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2199fc0aeeef0203e54ad44ea3ac1a7, ASSIGN}] 2024-12-03T02:33:27,657 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2199fc0aeeef0203e54ad44ea3ac1a7, ASSIGN 2024-12-03T02:33:27,659 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2199fc0aeeef0203e54ad44ea3ac1a7, ASSIGN; state=OFFLINE, location=64035cbb0b46,38721,1733193202350; forceNewPlan=false, retain=false 2024-12-03T02:33:27,811 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a2199fc0aeeef0203e54ad44ea3ac1a7, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:27,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2199fc0aeeef0203e54ad44ea3ac1a7, ASSIGN because future has completed 2024-12-03T02:33:27,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:33:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:33:28,005 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:28,006 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:33:28,006 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,006 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:33:28,007 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,007 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,010 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,014 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:33:28,014 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2199fc0aeeef0203e54ad44ea3ac1a7 columnFamilyName A 2024-12-03T02:33:28,014 DEBUG [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:28,016 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] regionserver.HStore(327): Store=a2199fc0aeeef0203e54ad44ea3ac1a7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:28,016 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,019 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:33:28,019 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2199fc0aeeef0203e54ad44ea3ac1a7 columnFamilyName B 2024-12-03T02:33:28,019 DEBUG [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:28,020 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] regionserver.HStore(327): Store=a2199fc0aeeef0203e54ad44ea3ac1a7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:28,020 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,023 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:33:28,024 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2199fc0aeeef0203e54ad44ea3ac1a7 columnFamilyName C 2024-12-03T02:33:28,024 DEBUG [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:28,025 INFO [StoreOpener-a2199fc0aeeef0203e54ad44ea3ac1a7-1 {}] regionserver.HStore(327): Store=a2199fc0aeeef0203e54ad44ea3ac1a7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:28,026 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:28,026 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,027 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,028 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,030 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,030 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,031 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:33:28,034 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,038 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:33:28,040 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a2199fc0aeeef0203e54ad44ea3ac1a7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67486456, jitterRate=0.005626559257507324}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:33:28,040 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:28,041 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a2199fc0aeeef0203e54ad44ea3ac1a7: Running coprocessor pre-open hook at 1733193208007Writing region info on filesystem at 1733193208007Initializing all the Stores at 1733193208009 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193208009Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193208009Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193208010 (+1 ms)Cleaning up temporary data from old regions at 1733193208030 (+20 ms)Running coprocessor post-open hooks at 1733193208040 (+10 ms)Region opened successfully at 1733193208041 (+1 ms) 2024-12-03T02:33:28,044 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., pid=6, masterSystemTime=1733193207999 2024-12-03T02:33:28,047 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:28,047 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:28,049 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a2199fc0aeeef0203e54ad44ea3ac1a7, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:28,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:33:28,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T02:33:28,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 in 217 msec 2024-12-03T02:33:28,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T02:33:28,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2199fc0aeeef0203e54ad44ea3ac1a7, ASSIGN in 407 msec 2024-12-03T02:33:28,070 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T02:33:28,070 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193208070"}]},"ts":"1733193208070"} 2024-12-03T02:33:28,074 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T02:33:28,077 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T02:33:28,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3080 sec 2024-12-03T02:33:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:33:28,957 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:28,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:28,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:28,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:28,973 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508e5ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:28,978 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fe15c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:28,981 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:28,994 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:28,998 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T02:33:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 64035cbb0b46,44739,1733193201066 2024-12-03T02:33:29,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@253960f5 2024-12-03T02:33:29,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T02:33:29,010 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T02:33:29,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:29,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:29,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395f1c25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:29,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:29,028 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:29,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:29,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:29,035 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd026af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:29,039 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:29,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:29,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:29,044 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:29,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c5f854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:29,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:29,049 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:29,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1768b9d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:29,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:29,058 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d903666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:29,062 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:29,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59441750, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:29,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:29,066 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b904aa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:29,068 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:29,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df19f74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:29,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:29,073 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6891f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:29,077 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe9eb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:29,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:29,080 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:29,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:29,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:29,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa07968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:29,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:29,083 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5097bf60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:29,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:29,087 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12dac1a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:29,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:29,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:29,090 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:29,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce25cec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:29,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:29,097 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6034ec4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:29,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:29,100 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6f2640, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:29,105 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:29,105 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:29,105 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:29,105 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:29,105 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:29,108 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:29,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:29,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:29,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:29,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:29,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:29,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:29,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:29,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:33:29,116 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:29,116 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:29,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:29,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:29,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:33:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-03T02:33:29,126 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:29,126 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:33:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T02:33:29,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:29,129 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:33:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:33:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:29,132 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:29,132 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:29,132 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:29,135 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:33:29,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:33:29,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:33:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:33:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:33:29,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:33:29,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:33:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:33:29,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:33:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:29,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T02:33:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:29,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:29,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:29,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:29,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:29,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/53cc22542fb340089bd241ccae2104b0 is 50, key is test_row_0/A:col10/1733193209206/Put/seqid=0 2024-12-03T02:33:29,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193209295, max timestamp=1733193209381Num uniques 859; 2024-12-03T02:33:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193269364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193269369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:33:29,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193209295, max timestamp=1733193209379Num uniques 859; 2024-12-03T02:33:29,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:33:29,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193209295, max timestamp=1733193209381Num uniques 859; 2024-12-03T02:33:29,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:33:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T02:33:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741837_1013 (size=12001) 2024-12-03T02:33:29,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:29,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:29,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:29,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:29,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193269552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193269569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193269572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193269580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193269603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193269676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193269705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193269716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:29,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T02:33:29,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193269806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193269816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/53cc22542fb340089bd241ccae2104b0 2024-12-03T02:33:29,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193269896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:29,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:29,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193269925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:29,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193269926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/971906c5209844c8903c23baab9c2d97 is 50, key is test_row_0/B:col10/1733193209206/Put/seqid=0 2024-12-03T02:33:30,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741838_1014 (size=12001) 2024-12-03T02:33:30,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/971906c5209844c8903c23baab9c2d97 2024-12-03T02:33:30,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:30,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:30,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/535b70fcce7047b7aac7f8acb9d2b4db is 50, key is test_row_0/C:col10/1733193209206/Put/seqid=0 2024-12-03T02:33:30,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193270135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193270131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741839_1015 (size=12001) 2024-12-03T02:33:30,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193270208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:30,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:30,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193270244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193270248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T02:33:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/535b70fcce7047b7aac7f8acb9d2b4db 2024-12-03T02:33:30,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/53cc22542fb340089bd241ccae2104b0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/53cc22542fb340089bd241ccae2104b0 2024-12-03T02:33:30,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/53cc22542fb340089bd241ccae2104b0, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T02:33:30,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/971906c5209844c8903c23baab9c2d97 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/971906c5209844c8903c23baab9c2d97 2024-12-03T02:33:30,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/971906c5209844c8903c23baab9c2d97, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T02:33:30,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/535b70fcce7047b7aac7f8acb9d2b4db as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/535b70fcce7047b7aac7f8acb9d2b4db 2024-12-03T02:33:30,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:30,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:30,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193270656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193270656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193270727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/535b70fcce7047b7aac7f8acb9d2b4db, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T02:33:30,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 1557ms, sequenceid=12, compaction requested=false 2024-12-03T02:33:30,771 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T02:33:30,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:30,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T02:33:30,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:30,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:30,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:30,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:30,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:30,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:30,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/cda043a1adc54167a032f336b682c406 is 50, key is test_row_0/A:col10/1733193210789/Put/seqid=0 2024-12-03T02:33:30,870 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T02:33:30,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:30,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:30,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:30,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:30,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193270878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741840_1016 (size=14341) 2024-12-03T02:33:30,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193270899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:30,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/cda043a1adc54167a032f336b682c406 2024-12-03T02:33:30,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/270b32f8c00b4753afaa26598309fe67 is 50, key is test_row_0/B:col10/1733193210789/Put/seqid=0 2024-12-03T02:33:31,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193271015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:31,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193271027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:31,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741841_1017 (size=12001) 2024-12-03T02:33:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:31,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:31,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/270b32f8c00b4753afaa26598309fe67 2024-12-03T02:33:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:31,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:31,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:31,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/861cd31d00f14bd5ac90098662470204 is 50, key is test_row_0/C:col10/1733193210789/Put/seqid=0 2024-12-03T02:33:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741842_1018 (size=12001) 2024-12-03T02:33:31,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/861cd31d00f14bd5ac90098662470204 2024-12-03T02:33:31,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:31,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:31,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:31,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:31,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:31,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/cda043a1adc54167a032f336b682c406 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/cda043a1adc54167a032f336b682c406 2024-12-03T02:33:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:31,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193271235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:31,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/cda043a1adc54167a032f336b682c406, entries=200, sequenceid=49, filesize=14.0 K 2024-12-03T02:33:31,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193271256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:31,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/270b32f8c00b4753afaa26598309fe67 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/270b32f8c00b4753afaa26598309fe67 2024-12-03T02:33:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T02:33:31,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/270b32f8c00b4753afaa26598309fe67, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T02:33:31,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/861cd31d00f14bd5ac90098662470204 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/861cd31d00f14bd5ac90098662470204 2024-12-03T02:33:31,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/861cd31d00f14bd5ac90098662470204, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T02:33:31,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 565ms, sequenceid=49, compaction requested=false 2024-12-03T02:33:31,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:31,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T02:33:31,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:31,393 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T02:33:31,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:31,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/54f14c14520742c78037eaae62735227 is 50, key is test_row_0/A:col10/1733193210860/Put/seqid=0 2024-12-03T02:33:31,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741843_1019 (size=12001) 2024-12-03T02:33:31,442 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/54f14c14520742c78037eaae62735227 2024-12-03T02:33:31,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/a312dcf912ad4891a98da1e3b89a13ef is 50, key is test_row_0/B:col10/1733193210860/Put/seqid=0 2024-12-03T02:33:31,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741844_1020 (size=12001) 2024-12-03T02:33:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:31,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:31,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193211708, max timestamp=1733193211739Num uniques 859; 2024-12-03T02:33:31,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:33:31,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193211708, max timestamp=1733193211739Num uniques 859; 2024-12-03T02:33:31,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:33:31,771 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:33:31,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193211708, max timestamp=1733193211739Num uniques 859; 2024-12-03T02:33:31,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:33:31,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193211795, max timestamp=1733193211856Num uniques 1018; 2024-12-03T02:33:31,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T02:33:31,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193211795, max timestamp=1733193211856Num uniques 1018; 2024-12-03T02:33:31,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T02:33:31,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193211795, max timestamp=1733193211856Num uniques 1009; 2024-12-03T02:33:31,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-03T02:33:31,914 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/a312dcf912ad4891a98da1e3b89a13ef 2024-12-03T02:33:31,942 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T02:33:31,942 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T02:33:31,945 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T02:33:31,945 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T02:33:31,945 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T02:33:31,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T02:33:31,947 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T02:33:31,947 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T02:33:31,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/193ae69d79074b18ae33fc053a47cb89 is 50, key is test_row_0/C:col10/1733193210860/Put/seqid=0 2024-12-03T02:33:32,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741845_1021 (size=12001) 2024-12-03T02:33:32,008 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/193ae69d79074b18ae33fc053a47cb89 2024-12-03T02:33:32,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193271985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193271994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/54f14c14520742c78037eaae62735227 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54f14c14520742c78037eaae62735227 2024-12-03T02:33:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193272002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193272010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733193211986, max timestamp=1733193212005Num uniques 1077; 2024-12-03T02:33:32,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-03T02:33:32,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733193211986, max timestamp=1733193212005Num uniques 1009; 2024-12-03T02:33:32,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-03T02:33:32,054 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54f14c14520742c78037eaae62735227, entries=150, sequenceid=58, filesize=11.7 K 2024-12-03T02:33:32,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733193211986, max timestamp=1733193212005Num uniques 1077; 2024-12-03T02:33:32,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-03T02:33:32,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/a312dcf912ad4891a98da1e3b89a13ef as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/a312dcf912ad4891a98da1e3b89a13ef 2024-12-03T02:33:32,075 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/a312dcf912ad4891a98da1e3b89a13ef, entries=150, sequenceid=58, filesize=11.7 K 2024-12-03T02:33:32,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/193ae69d79074b18ae33fc053a47cb89 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/193ae69d79074b18ae33fc053a47cb89 2024-12-03T02:33:32,111 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/193ae69d79074b18ae33fc053a47cb89, entries=150, sequenceid=58, filesize=11.7 K 2024-12-03T02:33:32,115 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=46.96 KB/48090 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 721ms, sequenceid=58, compaction requested=true 2024-12-03T02:33:32,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:32,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:32,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-03T02:33:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-03T02:33:32,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T02:33:32,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9820 sec 2024-12-03T02:33:32,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.0090 sec 2024-12-03T02:33:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:32,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=53.67 KB heapSize=144.23 KB 2024-12-03T02:33:32,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:32,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:32,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:32,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:32,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:32,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:32,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/1a5025f86f68425a922fcf22c4d29e73 is 50, key is test_row_0/A:col10/1733193212123/Put/seqid=0 2024-12-03T02:33:32,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741846_1022 (size=14341) 2024-12-03T02:33:32,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/1a5025f86f68425a922fcf22c4d29e73 2024-12-03T02:33:32,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/0537fc216601434e8b165db152f7bb57 is 50, key is test_row_0/B:col10/1733193212123/Put/seqid=0 2024-12-03T02:33:32,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193212262, max timestamp=1733193212295Num uniques 859; 2024-12-03T02:33:32,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:32,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193212274, max timestamp=1733193212295Num uniques 859; 2024-12-03T02:33:32,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:33:32,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193212274, max timestamp=1733193212295Num uniques 859; 2024-12-03T02:33:32,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:33:32,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741847_1023 (size=12001) 2024-12-03T02:33:32,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/0537fc216601434e8b165db152f7bb57 2024-12-03T02:33:32,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/eb908facbe8744dc9e0f3c4551e97d96 is 50, key is test_row_0/C:col10/1733193212123/Put/seqid=0 2024-12-03T02:33:32,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193272417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193272421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193272413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193272431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741848_1024 (size=12001) 2024-12-03T02:33:32,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/eb908facbe8744dc9e0f3c4551e97d96 2024-12-03T02:33:32,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193272442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/1a5025f86f68425a922fcf22c4d29e73 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/1a5025f86f68425a922fcf22c4d29e73 2024-12-03T02:33:32,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193272556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/1a5025f86f68425a922fcf22c4d29e73, entries=200, sequenceid=117, filesize=14.0 K 2024-12-03T02:33:32,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193272567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193272566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193272567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/0537fc216601434e8b165db152f7bb57 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0537fc216601434e8b165db152f7bb57 2024-12-03T02:33:32,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193272595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0537fc216601434e8b165db152f7bb57, entries=150, sequenceid=117, filesize=11.7 K 2024-12-03T02:33:32,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/eb908facbe8744dc9e0f3c4551e97d96 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/eb908facbe8744dc9e0f3c4551e97d96 2024-12-03T02:33:32,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/eb908facbe8744dc9e0f3c4551e97d96, entries=150, sequenceid=117, filesize=11.7 K 2024-12-03T02:33:32,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~144.19 KB/147648, currentSize=131.94 KB/135110 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 585ms, sequenceid=117, compaction requested=true 2024-12-03T02:33:32,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:32,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:32,737 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:32,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:32,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:32,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:32,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:32,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:33:32,751 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:32,757 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:32,762 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:32,762 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/B is initiating minor compaction (all files) 2024-12-03T02:33:32,765 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/A is initiating minor compaction (all files) 2024-12-03T02:33:32,769 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/B in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:32,770 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/971906c5209844c8903c23baab9c2d97, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/270b32f8c00b4753afaa26598309fe67, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/a312dcf912ad4891a98da1e3b89a13ef, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0537fc216601434e8b165db152f7bb57] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=46.9 K 2024-12-03T02:33:32,773 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 971906c5209844c8903c23baab9c2d97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733193209182 2024-12-03T02:33:32,775 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/A in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:32,775 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/53cc22542fb340089bd241ccae2104b0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/cda043a1adc54167a032f336b682c406, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54f14c14520742c78037eaae62735227, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/1a5025f86f68425a922fcf22c4d29e73] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=51.4 K 2024-12-03T02:33:32,775 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 270b32f8c00b4753afaa26598309fe67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733193209541 2024-12-03T02:33:32,784 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting a312dcf912ad4891a98da1e3b89a13ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733193210837 2024-12-03T02:33:32,788 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 0537fc216601434e8b165db152f7bb57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733193212005 2024-12-03T02:33:32,789 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53cc22542fb340089bd241ccae2104b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733193209182 2024-12-03T02:33:32,799 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting cda043a1adc54167a032f336b682c406, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733193209541 2024-12-03T02:33:32,802 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f14c14520742c78037eaae62735227, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733193210837 2024-12-03T02:33:32,804 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5025f86f68425a922fcf22c4d29e73, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733193212005 2024-12-03T02:33:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:32,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-03T02:33:32,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:32,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:32,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:32,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:32,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:32,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:32,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/7917efbf10e048159ba07980c2462114 is 50, key is test_row_0/A:col10/1733193212793/Put/seqid=0 2024-12-03T02:33:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193272877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193272883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,910 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#A#compaction#13 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:32,910 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/8a5b27458b3046f7b53001f6d59bb405 is 50, key is test_row_0/A:col10/1733193212123/Put/seqid=0 2024-12-03T02:33:32,912 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#B#compaction#14 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:32,912 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/ac1454a1e20e4cd3b5e8c39ce49466a9 is 50, key is test_row_0/B:col10/1733193212123/Put/seqid=0 2024-12-03T02:33:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193272890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193272895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193272902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:32,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741849_1025 (size=14541) 2024-12-03T02:33:32,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741850_1026 (size=12139) 2024-12-03T02:33:33,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741851_1027 (size=12139) 2024-12-03T02:33:33,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193273005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193273025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193273035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,055 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/ac1454a1e20e4cd3b5e8c39ce49466a9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ac1454a1e20e4cd3b5e8c39ce49466a9 2024-12-03T02:33:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193273036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193273037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,089 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/B of a2199fc0aeeef0203e54ad44ea3ac1a7 into ac1454a1e20e4cd3b5e8c39ce49466a9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:33,090 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:33,099 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/B, priority=12, startTime=1733193212749; duration=0sec 2024-12-03T02:33:33,103 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:33,104 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:B 2024-12-03T02:33:33,104 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:33,121 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:33,121 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/C is initiating minor compaction (all files) 2024-12-03T02:33:33,122 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/C in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:33,122 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/535b70fcce7047b7aac7f8acb9d2b4db, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/861cd31d00f14bd5ac90098662470204, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/193ae69d79074b18ae33fc053a47cb89, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/eb908facbe8744dc9e0f3c4551e97d96] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=46.9 K 2024-12-03T02:33:33,126 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 535b70fcce7047b7aac7f8acb9d2b4db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733193209182 2024-12-03T02:33:33,132 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 861cd31d00f14bd5ac90098662470204, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733193209541 2024-12-03T02:33:33,133 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 193ae69d79074b18ae33fc053a47cb89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733193210837 2024-12-03T02:33:33,134 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting eb908facbe8744dc9e0f3c4551e97d96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733193212005 2024-12-03T02:33:33,192 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#C#compaction#15 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:33,193 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/27317faa5cd544a38be56f9910c7b198 is 50, key is test_row_0/C:col10/1733193212123/Put/seqid=0 2024-12-03T02:33:33,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741852_1028 (size=12139) 2024-12-03T02:33:33,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193273236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193273245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193273266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193273265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193273275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T02:33:33,308 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:33,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:33:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-03T02:33:33,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:33:33,337 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:33:33,338 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:33:33,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:33:33,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/7917efbf10e048159ba07980c2462114 2024-12-03T02:33:33,423 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/8a5b27458b3046f7b53001f6d59bb405 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8a5b27458b3046f7b53001f6d59bb405 2024-12-03T02:33:33,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/42ae3beeb5af42d89cc4cac3227b7460 is 50, key is test_row_0/B:col10/1733193212793/Put/seqid=0 2024-12-03T02:33:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:33:33,452 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/A of a2199fc0aeeef0203e54ad44ea3ac1a7 into 8a5b27458b3046f7b53001f6d59bb405(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:33,452 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:33,452 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/A, priority=12, startTime=1733193212722; duration=0sec 2024-12-03T02:33:33,461 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:33,461 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:A 2024-12-03T02:33:33,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741853_1029 (size=12151) 2024-12-03T02:33:33,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/42ae3beeb5af42d89cc4cac3227b7460 2024-12-03T02:33:33,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T02:33:33,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:33,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:33,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:33,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:33,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:33,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/be57be4a845148d89420c336e3e54a6f is 50, key is test_row_0/C:col10/1733193212793/Put/seqid=0 2024-12-03T02:33:33,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193273565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193273565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741854_1030 (size=12151) 2024-12-03T02:33:33,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193273595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193273596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193273597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:33:33,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T02:33:33,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:33,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:33,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:33,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:33,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:33,672 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/27317faa5cd544a38be56f9910c7b198 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/27317faa5cd544a38be56f9910c7b198 2024-12-03T02:33:33,692 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/C of a2199fc0aeeef0203e54ad44ea3ac1a7 into 27317faa5cd544a38be56f9910c7b198(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:33,692 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:33,692 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/C, priority=12, startTime=1733193212749; duration=0sec 2024-12-03T02:33:33,692 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:33,692 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:C 2024-12-03T02:33:33,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T02:33:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:33,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:33:33,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/be57be4a845148d89420c336e3e54a6f 2024-12-03T02:33:34,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T02:33:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:34,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:34,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:34,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/7917efbf10e048159ba07980c2462114 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7917efbf10e048159ba07980c2462114 2024-12-03T02:33:34,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7917efbf10e048159ba07980c2462114, entries=200, sequenceid=154, filesize=14.2 K 2024-12-03T02:33:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/42ae3beeb5af42d89cc4cac3227b7460 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/42ae3beeb5af42d89cc4cac3227b7460 2024-12-03T02:33:34,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193274085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:34,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193274091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:34,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/42ae3beeb5af42d89cc4cac3227b7460, entries=150, sequenceid=154, filesize=11.9 K 2024-12-03T02:33:34,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/be57be4a845148d89420c336e3e54a6f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/be57be4a845148d89420c336e3e54a6f 2024-12-03T02:33:34,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:34,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193274115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:34,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193274127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193274136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:34,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/be57be4a845148d89420c336e3e54a6f, entries=150, sequenceid=154, filesize=11.9 K 2024-12-03T02:33:34,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T02:33:34,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:34,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:34,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:34,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:34,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:34,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:34,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 1358ms, sequenceid=154, compaction requested=false 2024-12-03T02:33:34,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:34,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T02:33:34,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:34,332 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T02:33:34,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:34,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:34,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:34,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:34,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/b6fc4e833bfc403d8400b31c6efb8006 is 50, key is test_row_0/A:col10/1733193212891/Put/seqid=0 2024-12-03T02:33:34,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741855_1031 (size=12151) 2024-12-03T02:33:34,416 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/b6fc4e833bfc403d8400b31c6efb8006 2024-12-03T02:33:34,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/ba34090c9d054077bf15dbee468e096c is 50, key is test_row_0/B:col10/1733193212891/Put/seqid=0 2024-12-03T02:33:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:33:34,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741856_1032 (size=12151) 2024-12-03T02:33:34,937 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/ba34090c9d054077bf15dbee468e096c 2024-12-03T02:33:35,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/db459e52eb834626857050c7abf094e6 is 50, key is test_row_0/C:col10/1733193212891/Put/seqid=0 2024-12-03T02:33:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:35,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:35,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741857_1033 (size=12151) 2024-12-03T02:33:35,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733193215182, max timestamp=1733193215217Num uniques 859; 2024-12-03T02:33:35,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T02:33:35,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733193215182, max timestamp=1733193215217Num uniques 859; 2024-12-03T02:33:35,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T02:33:35,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733193215182, max timestamp=1733193215217Num uniques 859; 2024-12-03T02:33:35,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T02:33:35,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193275263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193275267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193275267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193275277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193275277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193275392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193275396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193275396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193275399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193275419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:33:35,551 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/db459e52eb834626857050c7abf094e6 2024-12-03T02:33:35,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/b6fc4e833bfc403d8400b31c6efb8006 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/b6fc4e833bfc403d8400b31c6efb8006 2024-12-03T02:33:35,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193275608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193275609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193275609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,633 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/b6fc4e833bfc403d8400b31c6efb8006, entries=150, sequenceid=166, filesize=11.9 K 2024-12-03T02:33:35,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/ba34090c9d054077bf15dbee468e096c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ba34090c9d054077bf15dbee468e096c 2024-12-03T02:33:35,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193275636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193275625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:35,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:35,667 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ba34090c9d054077bf15dbee468e096c, entries=150, sequenceid=166, filesize=11.9 K 2024-12-03T02:33:35,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/db459e52eb834626857050c7abf094e6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/db459e52eb834626857050c7abf094e6 2024-12-03T02:33:35,685 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/db459e52eb834626857050c7abf094e6, entries=150, sequenceid=166, filesize=11.9 K 2024-12-03T02:33:35,688 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=154.31 KB/158010 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 1355ms, sequenceid=166, compaction requested=true 2024-12-03T02:33:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-03T02:33:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-03T02:33:35,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-03T02:33:35,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3630 sec 2024-12-03T02:33:35,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.3960 sec 2024-12-03T02:33:35,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193215247, max timestamp=1733193215946Num uniques 1168; 2024-12-03T02:33:35,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-12-03T02:33:35,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193215247, max timestamp=1733193215948Num uniques 1168; 2024-12-03T02:33:35,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193215247, max timestamp=1733193215948Num uniques 1168; 2024-12-03T02:33:35,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-12-03T02:33:35,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-12-03T02:33:35,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=60.38 KB heapSize=165 KB 2024-12-03T02:33:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:35,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:35,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:35,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:36,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/c61554c61562494cb2d19c3ff3d48558 is 50, key is test_row_0/A:col10/1733193215985/Put/seqid=0 2024-12-03T02:33:36,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741858_1034 (size=12151) 2024-12-03T02:33:36,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193216058, max timestamp=1733193216070Num uniques 859; 2024-12-03T02:33:36,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193216058, max timestamp=1733193216070Num uniques 859; 2024-12-03T02:33:36,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:33:36,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:33:36,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193216058, max timestamp=1733193216070Num uniques 859; 2024-12-03T02:33:36,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:33:36,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193276154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193276151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193276169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193276173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193276173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193276286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193276287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193276296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193276296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193276296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/c61554c61562494cb2d19c3ff3d48558 2024-12-03T02:33:36,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/43704ccbad7f4caf90f820d6d08331be is 50, key is test_row_0/B:col10/1733193215985/Put/seqid=0 2024-12-03T02:33:36,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193276505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193276506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193276516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741859_1035 (size=12151) 2024-12-03T02:33:36,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/43704ccbad7f4caf90f820d6d08331be 2024-12-03T02:33:36,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193276535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193276536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/29ec8a0bbd544feeb7f1cd3b60eaa633 is 50, key is test_row_0/C:col10/1733193215985/Put/seqid=0 2024-12-03T02:33:36,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741860_1036 (size=12151) 2024-12-03T02:33:36,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/29ec8a0bbd544feeb7f1cd3b60eaa633 2024-12-03T02:33:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/c61554c61562494cb2d19c3ff3d48558 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c61554c61562494cb2d19c3ff3d48558 2024-12-03T02:33:36,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c61554c61562494cb2d19c3ff3d48558, entries=150, sequenceid=210, filesize=11.9 K 2024-12-03T02:33:36,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/43704ccbad7f4caf90f820d6d08331be as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/43704ccbad7f4caf90f820d6d08331be 2024-12-03T02:33:36,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/43704ccbad7f4caf90f820d6d08331be, entries=150, sequenceid=210, filesize=11.9 K 2024-12-03T02:33:36,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/29ec8a0bbd544feeb7f1cd3b60eaa633 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/29ec8a0bbd544feeb7f1cd3b60eaa633 2024-12-03T02:33:36,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/29ec8a0bbd544feeb7f1cd3b60eaa633, entries=150, sequenceid=210, filesize=11.9 K 2024-12-03T02:33:36,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~184.88 KB/189312, currentSize=127.47 KB/130530 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 745ms, sequenceid=210, compaction requested=true 2024-12-03T02:33:36,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:36,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:36,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:36,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:36,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:33:36,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:36,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T02:33:36,731 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:36,732 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:36,735 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:36,735 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50982 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:36,735 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/C is initiating minor compaction (all files) 2024-12-03T02:33:36,735 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/A is initiating minor compaction (all files) 2024-12-03T02:33:36,735 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/C in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:36,735 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/A in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:36,735 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8a5b27458b3046f7b53001f6d59bb405, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7917efbf10e048159ba07980c2462114, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/b6fc4e833bfc403d8400b31c6efb8006, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c61554c61562494cb2d19c3ff3d48558] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=49.8 K 2024-12-03T02:33:36,735 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/27317faa5cd544a38be56f9910c7b198, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/be57be4a845148d89420c336e3e54a6f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/db459e52eb834626857050c7abf094e6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/29ec8a0bbd544feeb7f1cd3b60eaa633] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=47.5 K 2024-12-03T02:33:36,736 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 27317faa5cd544a38be56f9910c7b198, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733193212005 2024-12-03T02:33:36,736 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a5b27458b3046f7b53001f6d59bb405, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733193212005 2024-12-03T02:33:36,737 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7917efbf10e048159ba07980c2462114, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733193212416 2024-12-03T02:33:36,738 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting be57be4a845148d89420c336e3e54a6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733193212416 2024-12-03T02:33:36,738 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting db459e52eb834626857050c7abf094e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733193212886 2024-12-03T02:33:36,739 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6fc4e833bfc403d8400b31c6efb8006, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733193212886 2024-12-03T02:33:36,739 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 29ec8a0bbd544feeb7f1cd3b60eaa633, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733193215971 2024-12-03T02:33:36,741 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting c61554c61562494cb2d19c3ff3d48558, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733193215971 2024-12-03T02:33:36,794 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#A#compaction#24 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:36,796 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/d1abae5bd7794876802532e8284846f3 is 50, key is test_row_0/A:col10/1733193215985/Put/seqid=0 2024-12-03T02:33:36,798 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#C#compaction#25 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:36,799 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/5df1f8d81cee4a9dbfed5dd6e7eee8af is 50, key is test_row_0/C:col10/1733193215985/Put/seqid=0 2024-12-03T02:33:36,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741861_1037 (size=12425) 2024-12-03T02:33:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:36,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T02:33:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:36,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741862_1038 (size=12425) 2024-12-03T02:33:36,867 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/d1abae5bd7794876802532e8284846f3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/d1abae5bd7794876802532e8284846f3 2024-12-03T02:33:36,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/712c378865974b06ba931da8e32a129f is 50, key is test_row_0/A:col10/1733193216836/Put/seqid=0 2024-12-03T02:33:36,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193276857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,881 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/5df1f8d81cee4a9dbfed5dd6e7eee8af as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/5df1f8d81cee4a9dbfed5dd6e7eee8af 2024-12-03T02:33:36,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193276866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193276868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193276871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193276874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,893 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/A of a2199fc0aeeef0203e54ad44ea3ac1a7 into d1abae5bd7794876802532e8284846f3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:36,894 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:36,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,895 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/C of a2199fc0aeeef0203e54ad44ea3ac1a7 into 5df1f8d81cee4a9dbfed5dd6e7eee8af(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:36,895 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:36,895 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/C, priority=12, startTime=1733193216731; duration=0sec 2024-12-03T02:33:36,895 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:36,895 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:C 2024-12-03T02:33:36,895 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:36,896 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/A, priority=12, startTime=1733193216731; duration=0sec 2024-12-03T02:33:36,896 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:36,896 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:A 2024-12-03T02:33:36,898 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:36,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741863_1039 (size=14541) 2024-12-03T02:33:36,899 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/B is initiating minor compaction (all files) 2024-12-03T02:33:36,899 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/B in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:36,899 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ac1454a1e20e4cd3b5e8c39ce49466a9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/42ae3beeb5af42d89cc4cac3227b7460, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ba34090c9d054077bf15dbee468e096c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/43704ccbad7f4caf90f820d6d08331be] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=47.5 K 2024-12-03T02:33:36,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/712c378865974b06ba931da8e32a129f 2024-12-03T02:33:36,902 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ac1454a1e20e4cd3b5e8c39ce49466a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733193212005 2024-12-03T02:33:36,920 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 42ae3beeb5af42d89cc4cac3227b7460, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733193212416 2024-12-03T02:33:36,950 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ba34090c9d054077bf15dbee468e096c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733193212886 2024-12-03T02:33:36,952 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 43704ccbad7f4caf90f820d6d08331be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733193215971 2024-12-03T02:33:36,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193276987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:36,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/777473178e4e419d9727bc7f2a9bd6b1 is 50, key is test_row_0/B:col10/1733193216836/Put/seqid=0 2024-12-03T02:33:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193276997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193276997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,007 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,008 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/aa8048d794f54db09e00e44f5289e8c7 is 50, key is test_row_0/B:col10/1733193215985/Put/seqid=0 2024-12-03T02:33:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193277006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193277008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741865_1041 (size=12425) 2024-12-03T02:33:37,047 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/aa8048d794f54db09e00e44f5289e8c7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/aa8048d794f54db09e00e44f5289e8c7 2024-12-03T02:33:37,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741864_1040 (size=12151) 2024-12-03T02:33:37,072 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/B of a2199fc0aeeef0203e54ad44ea3ac1a7 into aa8048d794f54db09e00e44f5289e8c7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:37,072 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:37,072 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/B, priority=12, startTime=1733193216731; duration=0sec 2024-12-03T02:33:37,072 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:37,072 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:B 2024-12-03T02:33:37,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193277205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193277225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193277231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193277231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193277232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/777473178e4e419d9727bc7f2a9bd6b1 2024-12-03T02:33:37,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/605a5a1704d244b8905e641e7a55d981 is 50, key is test_row_0/C:col10/1733193216836/Put/seqid=0 2024-12-03T02:33:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:33:37,515 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:37,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:33:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-03T02:33:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T02:33:37,523 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:33:37,526 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:33:37,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:33:37,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193277528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193277545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193277546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193277546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193277546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:37,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741866_1042 (size=12151) 2024-12-03T02:33:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:37,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:37,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/605a5a1704d244b8905e641e7a55d981 2024-12-03T02:33:37,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/712c378865974b06ba931da8e32a129f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/712c378865974b06ba931da8e32a129f 2024-12-03T02:33:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T02:33:37,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/712c378865974b06ba931da8e32a129f, entries=200, sequenceid=247, filesize=14.2 K 2024-12-03T02:33:37,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/777473178e4e419d9727bc7f2a9bd6b1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/777473178e4e419d9727bc7f2a9bd6b1 2024-12-03T02:33:37,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T02:33:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:37,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:37,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/777473178e4e419d9727bc7f2a9bd6b1, entries=150, sequenceid=247, filesize=11.9 K 2024-12-03T02:33:37,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/605a5a1704d244b8905e641e7a55d981 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/605a5a1704d244b8905e641e7a55d981 2024-12-03T02:33:37,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/605a5a1704d244b8905e641e7a55d981, entries=150, sequenceid=247, filesize=11.9 K 2024-12-03T02:33:37,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 916ms, sequenceid=247, compaction requested=false 2024-12-03T02:33:37,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:37,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T02:33:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:37,837 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T02:33:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:37,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:37,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:37,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T02:33:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/54b1b109862f422d9a895babc0c35785 is 50, key is test_row_0/A:col10/1733193216865/Put/seqid=0 2024-12-03T02:33:37,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741867_1043 (size=12151) 2024-12-03T02:33:37,913 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/54b1b109862f422d9a895babc0c35785 2024-12-03T02:33:37,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/e6ecfcdd75f340d7ba27206593fc00a6 is 50, key is test_row_0/B:col10/1733193216865/Put/seqid=0 2024-12-03T02:33:37,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741868_1044 (size=12151) 2024-12-03T02:33:37,947 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/e6ecfcdd75f340d7ba27206593fc00a6 2024-12-03T02:33:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/ef7e75e2e9b640cbb64beb8209d2a319 is 50, key is test_row_0/C:col10/1733193216865/Put/seqid=0 2024-12-03T02:33:38,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741869_1045 (size=12151) 2024-12-03T02:33:38,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T02:33:38,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193218181, max timestamp=1733193218208Num uniques 859; 2024-12-03T02:33:38,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:38,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193218181, max timestamp=1733193218208Num uniques 859; 2024-12-03T02:33:38,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:38,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193218189, max timestamp=1733193218208Num uniques 859; 2024-12-03T02:33:38,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:33:38,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193218268, max timestamp=1733193218289Num uniques 1118; 2024-12-03T02:33:38,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193218268, max timestamp=1733193218289Num uniques 1018; 2024-12-03T02:33:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193218268, max timestamp=1733193218288Num uniques 1118; 2024-12-03T02:33:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-03T02:33:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-03T02:33:38,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T02:33:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193278326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193278332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193278333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193278334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193278336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,434 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/ef7e75e2e9b640cbb64beb8209d2a319 2024-12-03T02:33:38,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/54b1b109862f422d9a895babc0c35785 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54b1b109862f422d9a895babc0c35785 2024-12-03T02:33:38,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193278446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193278447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193278447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193278447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,456 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54b1b109862f422d9a895babc0c35785, entries=150, sequenceid=259, filesize=11.9 K 2024-12-03T02:33:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/e6ecfcdd75f340d7ba27206593fc00a6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e6ecfcdd75f340d7ba27206593fc00a6 2024-12-03T02:33:38,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193278457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,475 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e6ecfcdd75f340d7ba27206593fc00a6, entries=150, sequenceid=259, filesize=11.9 K 2024-12-03T02:33:38,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/ef7e75e2e9b640cbb64beb8209d2a319 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ef7e75e2e9b640cbb64beb8209d2a319 2024-12-03T02:33:38,504 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ef7e75e2e9b640cbb64beb8209d2a319, entries=150, sequenceid=259, filesize=11.9 K 2024-12-03T02:33:38,509 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 672ms, sequenceid=259, compaction requested=true 2024-12-03T02:33:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:38,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-03T02:33:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-03T02:33:38,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T02:33:38,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 994 msec 2024-12-03T02:33:38,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0120 sec 2024-12-03T02:33:38,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-12-03T02:33:38,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:38,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:33:38,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:38,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:33:38,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:38,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:33:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T02:33:38,675 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:38,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:33:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-03T02:33:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:33:38,681 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:33:38,682 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:33:38,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:33:38,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/eee44da8ec9940d7b9eef404e86316fb is 50, key is test_row_0/A:col10/1733193218333/Put/seqid=0 2024-12-03T02:33:38,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193278681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193278686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193278690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193278695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193278697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741870_1046 (size=14741) 2024-12-03T02:33:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:33:38,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193278805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193278805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193278817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193278819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:38,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T02:33:38,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:38,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:38,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:38,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:38,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:38,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:33:39,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T02:33:39,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:39,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:39,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:39,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193279016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:39,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:39,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193279026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193279026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:39,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:39,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193279026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:39,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193279048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:39,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/eee44da8ec9940d7b9eef404e86316fb 2024-12-03T02:33:39,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T02:33:39,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:39,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/ad103dd4a89141a8885a9ec4f7a8b71c is 50, key is test_row_0/B:col10/1733193218333/Put/seqid=0 2024-12-03T02:33:39,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741871_1047 (size=12301) 2024-12-03T02:33:39,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/ad103dd4a89141a8885a9ec4f7a8b71c 2024-12-03T02:33:39,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/31b935d0169944e09a0236283fa0acc7 is 50, key is test_row_0/C:col10/1733193218333/Put/seqid=0 2024-12-03T02:33:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:33:39,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741872_1048 (size=12301) 2024-12-03T02:33:39,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T02:33:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:39,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/31b935d0169944e09a0236283fa0acc7 2024-12-03T02:33:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:39,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/eee44da8ec9940d7b9eef404e86316fb as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/eee44da8ec9940d7b9eef404e86316fb 2024-12-03T02:33:39,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193279353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:39,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:39,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193279354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:39,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/eee44da8ec9940d7b9eef404e86316fb, entries=200, sequenceid=314, filesize=14.4 K 2024-12-03T02:33:39,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193279355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:39,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/ad103dd4a89141a8885a9ec4f7a8b71c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ad103dd4a89141a8885a9ec4f7a8b71c 2024-12-03T02:33:39,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ad103dd4a89141a8885a9ec4f7a8b71c, entries=150, sequenceid=314, filesize=12.0 K 2024-12-03T02:33:39,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/31b935d0169944e09a0236283fa0acc7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/31b935d0169944e09a0236283fa0acc7 2024-12-03T02:33:39,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193279365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:39,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/31b935d0169944e09a0236283fa0acc7, entries=150, sequenceid=314, filesize=12.0 K 2024-12-03T02:33:39,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 752ms, sequenceid=314, compaction requested=true 2024-12-03T02:33:39,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:39,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:39,420 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:39,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:39,421 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:39,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:39,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:39,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:39,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:39,423 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53858 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:39,423 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/A is initiating minor compaction (all files) 2024-12-03T02:33:39,423 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/A in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,424 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/d1abae5bd7794876802532e8284846f3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/712c378865974b06ba931da8e32a129f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54b1b109862f422d9a895babc0c35785, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/eee44da8ec9940d7b9eef404e86316fb] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=52.6 K 2024-12-03T02:33:39,424 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49028 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:39,424 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/B is initiating minor compaction (all files) 2024-12-03T02:33:39,424 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1abae5bd7794876802532e8284846f3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733193215971 2024-12-03T02:33:39,424 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/B in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,424 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/aa8048d794f54db09e00e44f5289e8c7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/777473178e4e419d9727bc7f2a9bd6b1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e6ecfcdd75f340d7ba27206593fc00a6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ad103dd4a89141a8885a9ec4f7a8b71c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=47.9 K 2024-12-03T02:33:39,425 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 712c378865974b06ba931da8e32a129f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733193216150 2024-12-03T02:33:39,425 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting aa8048d794f54db09e00e44f5289e8c7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733193215971 2024-12-03T02:33:39,426 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54b1b109862f422d9a895babc0c35785, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733193216865 2024-12-03T02:33:39,428 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 777473178e4e419d9727bc7f2a9bd6b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733193216167 2024-12-03T02:33:39,428 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting eee44da8ec9940d7b9eef404e86316fb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733193218320 2024-12-03T02:33:39,429 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ecfcdd75f340d7ba27206593fc00a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733193216865 2024-12-03T02:33:39,430 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ad103dd4a89141a8885a9ec4f7a8b71c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733193218320 2024-12-03T02:33:39,467 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:39,468 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/95d1377ef56742bfbf7dd3ec7a10305e is 50, key is test_row_0/A:col10/1733193218333/Put/seqid=0 2024-12-03T02:33:39,483 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:39,484 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/302be0da1c4e418da5c81d80c6aa7cbf is 50, key is test_row_0/B:col10/1733193218333/Put/seqid=0 2024-12-03T02:33:39,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T02:33:39,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,490 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T02:33:39,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:39,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:39,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:39,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:39,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:39,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:39,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/15812b392cc54ca38caa5b2436980c08 is 50, key is test_row_0/A:col10/1733193218676/Put/seqid=0 2024-12-03T02:33:39,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741873_1049 (size=12711) 2024-12-03T02:33:39,554 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/95d1377ef56742bfbf7dd3ec7a10305e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/95d1377ef56742bfbf7dd3ec7a10305e 2024-12-03T02:33:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:39,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:39,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741874_1050 (size=12711) 2024-12-03T02:33:39,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741875_1051 (size=12301) 2024-12-03T02:33:39,594 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/A of a2199fc0aeeef0203e54ad44ea3ac1a7 into 95d1377ef56742bfbf7dd3ec7a10305e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:39,595 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:39,595 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/A, priority=12, startTime=1733193219420; duration=0sec 2024-12-03T02:33:39,595 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:39,595 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:A 2024-12-03T02:33:39,595 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:39,605 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49028 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:39,605 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/C is initiating minor compaction (all files) 2024-12-03T02:33:39,605 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/C in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:39,606 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/5df1f8d81cee4a9dbfed5dd6e7eee8af, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/605a5a1704d244b8905e641e7a55d981, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ef7e75e2e9b640cbb64beb8209d2a319, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/31b935d0169944e09a0236283fa0acc7] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=47.9 K 2024-12-03T02:33:39,608 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5df1f8d81cee4a9dbfed5dd6e7eee8af, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733193215971 2024-12-03T02:33:39,610 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 605a5a1704d244b8905e641e7a55d981, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733193216167 2024-12-03T02:33:39,614 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef7e75e2e9b640cbb64beb8209d2a319, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733193216865 2024-12-03T02:33:39,617 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31b935d0169944e09a0236283fa0acc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733193218320 2024-12-03T02:33:39,663 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:39,664 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/d86d302a5f4d486f91ceaf9de6f27eb3 is 50, key is test_row_0/C:col10/1733193218333/Put/seqid=0 2024-12-03T02:33:39,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741876_1052 (size=12711) 2024-12-03T02:33:39,723 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/d86d302a5f4d486f91ceaf9de6f27eb3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/d86d302a5f4d486f91ceaf9de6f27eb3 2024-12-03T02:33:39,744 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/C of a2199fc0aeeef0203e54ad44ea3ac1a7 into d86d302a5f4d486f91ceaf9de6f27eb3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:39,744 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:39,744 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/C, priority=12, startTime=1733193219421; duration=0sec 2024-12-03T02:33:39,745 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:39,745 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:C 2024-12-03T02:33:39,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193219630, max timestamp=1733193219782Num uniques 859; 2024-12-03T02:33:39,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193219630, max timestamp=1733193219782Num uniques 859; 2024-12-03T02:33:39,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:33:39,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:33:39,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193219630, max timestamp=1733193219782Num uniques 859; 2024-12-03T02:33:39,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:33:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:33:39,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193219886, max timestamp=1733193219904Num uniques 1018; 2024-12-03T02:33:39,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T02:33:39,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193219886, max timestamp=1733193219904Num uniques 1018; 2024-12-03T02:33:39,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193219886, max timestamp=1733193219904Num uniques 1018; 2024-12-03T02:33:39,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T02:33:39,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T02:33:39,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=477, cellSize=21842, totalHeapSize=49848, min timestamp=1733193219928, max timestamp=1733193219981Num uniques 1027; 2024-12-03T02:33:39,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=21842, suffix heap size=134776,new segment heap size=49848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=477 2024-12-03T02:33:39,988 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/15812b392cc54ca38caa5b2436980c08 2024-12-03T02:33:39,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8101, totalHeapSize=18648, min timestamp=1733193219956, max timestamp=1733193219986Num uniques 1027; 2024-12-03T02:33:39,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47031, new segment data size=8101, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T02:33:39,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733193219956, max timestamp=1733193219986Num uniques 1027; 2024-12-03T02:33:39,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T02:33:39,995 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/302be0da1c4e418da5c81d80c6aa7cbf as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/302be0da1c4e418da5c81d80c6aa7cbf 2024-12-03T02:33:40,013 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/B of a2199fc0aeeef0203e54ad44ea3ac1a7 into 302be0da1c4e418da5c81d80c6aa7cbf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:40,013 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:40,013 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/B, priority=12, startTime=1733193219421; duration=0sec 2024-12-03T02:33:40,013 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:40,013 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:B 2024-12-03T02:33:40,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/394838207210469ba7e2434680b12b8c is 50, key is test_row_0/B:col10/1733193218676/Put/seqid=0 2024-12-03T02:33:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193280050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193280053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193280054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193280058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193280060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741877_1053 (size=12301) 2024-12-03T02:33:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193280175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193280175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193280176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193280176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193280183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193280387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193280387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193280395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193280387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193280395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,483 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/394838207210469ba7e2434680b12b8c 2024-12-03T02:33:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/6da36179f470417087f729bfa44b9ba4 is 50, key is test_row_0/C:col10/1733193218676/Put/seqid=0 2024-12-03T02:33:40,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741878_1054 (size=12301) 2024-12-03T02:33:40,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193280711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193280711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193280709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193280709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193280712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:40,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:33:40,947 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/6da36179f470417087f729bfa44b9ba4 2024-12-03T02:33:40,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/15812b392cc54ca38caa5b2436980c08 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/15812b392cc54ca38caa5b2436980c08 2024-12-03T02:33:40,996 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/15812b392cc54ca38caa5b2436980c08, entries=150, sequenceid=322, filesize=12.0 K 2024-12-03T02:33:40,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/394838207210469ba7e2434680b12b8c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/394838207210469ba7e2434680b12b8c 2024-12-03T02:33:41,026 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/394838207210469ba7e2434680b12b8c, entries=150, sequenceid=322, filesize=12.0 K 2024-12-03T02:33:41,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/6da36179f470417087f729bfa44b9ba4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/6da36179f470417087f729bfa44b9ba4 2024-12-03T02:33:41,049 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/6da36179f470417087f729bfa44b9ba4, entries=150, sequenceid=322, filesize=12.0 K 2024-12-03T02:33:41,056 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 1566ms, sequenceid=322, compaction requested=false 2024-12-03T02:33:41,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:41,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:41,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-03T02:33:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-03T02:33:41,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T02:33:41,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3770 sec 2024-12-03T02:33:41,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.3940 sec 2024-12-03T02:33:41,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733193220049, max timestamp=1733193221228Num uniques 1036; 2024-12-03T02:33:41,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733193220049, max timestamp=1733193221228Num uniques 1336; 2024-12-03T02:33:41,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733193220049, max timestamp=1733193221228Num uniques 1036; 2024-12-03T02:33:41,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=61186, new segment data size=8516, suffix heap size=166912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1336, new segment cells count=186 2024-12-03T02:33:41,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T02:33:41,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T02:33:41,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:41,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=53.67 KB heapSize=143.39 KB 2024-12-03T02:33:41,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:41,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:41,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:41,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:41,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:41,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:41,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/2ba78ee35aec477e9ebeb439a16219b7 is 50, key is test_row_0/A:col10/1733193221239/Put/seqid=0 2024-12-03T02:33:41,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741879_1055 (size=12301) 2024-12-03T02:33:41,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/2ba78ee35aec477e9ebeb439a16219b7 2024-12-03T02:33:41,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193221281, max timestamp=1733193221296Num uniques 859; 2024-12-03T02:33:41,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193221281, max timestamp=1733193221296Num uniques 859; 2024-12-03T02:33:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193221281, max timestamp=1733193221296Num uniques 859; 2024-12-03T02:33:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:41,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/f6e11d8a2fcc417b96649c43cbd7a7da is 50, key is test_row_0/B:col10/1733193221239/Put/seqid=0 2024-12-03T02:33:41,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193281325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193281332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193281334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741880_1056 (size=12301) 2024-12-03T02:33:41,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/f6e11d8a2fcc417b96649c43cbd7a7da 2024-12-03T02:33:41,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193281340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193281341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/e48681d42e6c44159c5e06d8e2e8dc56 is 50, key is test_row_0/C:col10/1733193221239/Put/seqid=0 2024-12-03T02:33:41,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741881_1057 (size=12301) 2024-12-03T02:33:41,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/e48681d42e6c44159c5e06d8e2e8dc56 2024-12-03T02:33:41,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/2ba78ee35aec477e9ebeb439a16219b7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/2ba78ee35aec477e9ebeb439a16219b7 2024-12-03T02:33:41,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/2ba78ee35aec477e9ebeb439a16219b7, entries=150, sequenceid=401, filesize=12.0 K 2024-12-03T02:33:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/f6e11d8a2fcc417b96649c43cbd7a7da as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/f6e11d8a2fcc417b96649c43cbd7a7da 2024-12-03T02:33:41,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/f6e11d8a2fcc417b96649c43cbd7a7da, entries=150, sequenceid=401, filesize=12.0 K 2024-12-03T02:33:41,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/e48681d42e6c44159c5e06d8e2e8dc56 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e48681d42e6c44159c5e06d8e2e8dc56 2024-12-03T02:33:41,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e48681d42e6c44159c5e06d8e2e8dc56, entries=150, sequenceid=401, filesize=12.0 K 2024-12-03T02:33:41,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~143.34 KB/146784, currentSize=140.89 KB/144270 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 202ms, sequenceid=401, compaction requested=true 2024-12-03T02:33:41,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:41,444 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:41,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:41,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:41,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:41,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:41,446 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:41,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:41,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:41,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-03T02:33:41,450 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:41,450 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/A is initiating minor compaction (all files) 2024-12-03T02:33:41,450 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/A in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:41,451 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/95d1377ef56742bfbf7dd3ec7a10305e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/15812b392cc54ca38caa5b2436980c08, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/2ba78ee35aec477e9ebeb439a16219b7] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=36.4 K 2024-12-03T02:33:41,451 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:41,451 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/B is initiating minor compaction (all files) 2024-12-03T02:33:41,451 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/B in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:41,451 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/302be0da1c4e418da5c81d80c6aa7cbf, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/394838207210469ba7e2434680b12b8c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/f6e11d8a2fcc417b96649c43cbd7a7da] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=36.4 K 2024-12-03T02:33:41,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193221319, max timestamp=1733193221448Num uniques 1118; 2024-12-03T02:33:41,452 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 302be0da1c4e418da5c81d80c6aa7cbf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733193218320 2024-12-03T02:33:41,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193221319, max timestamp=1733193221448Num uniques 1118; 2024-12-03T02:33:41,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-03T02:33:41,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-03T02:33:41,452 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95d1377ef56742bfbf7dd3ec7a10305e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733193218320 2024-12-03T02:33:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:41,453 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 394838207210469ba7e2434680b12b8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733193218676 2024-12-03T02:33:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:41,453 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15812b392cc54ca38caa5b2436980c08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733193218676 2024-12-03T02:33:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:33:41,454 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting f6e11d8a2fcc417b96649c43cbd7a7da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733193220053 2024-12-03T02:33:41,454 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ba78ee35aec477e9ebeb439a16219b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733193220053 2024-12-03T02:33:41,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/e85d1ba9797c4d02bd97e0ba5f91155d is 50, key is test_row_0/A:col10/1733193221446/Put/seqid=0 2024-12-03T02:33:41,470 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:41,471 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/b385de93a41c409981c8cb6bea47b794 is 50, key is test_row_0/B:col10/1733193221239/Put/seqid=0 2024-12-03T02:33:41,474 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#A#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:41,475 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/4f94ed55141648c9a1c6aec104d24eea is 50, key is test_row_0/A:col10/1733193221239/Put/seqid=0 2024-12-03T02:33:41,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741882_1058 (size=14741) 2024-12-03T02:33:41,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/e85d1ba9797c4d02bd97e0ba5f91155d 2024-12-03T02:33:41,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193281488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193281489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193281491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741883_1059 (size=12813) 2024-12-03T02:33:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193281505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193281509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/25f8be2a822d4c2bad8bda1b8e9eba42 is 50, key is test_row_0/B:col10/1733193221446/Put/seqid=0 2024-12-03T02:33:41,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741884_1060 (size=12813) 2024-12-03T02:33:41,579 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/4f94ed55141648c9a1c6aec104d24eea as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/4f94ed55141648c9a1c6aec104d24eea 2024-12-03T02:33:41,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741885_1061 (size=12301) 2024-12-03T02:33:41,606 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/A of a2199fc0aeeef0203e54ad44ea3ac1a7 into 4f94ed55141648c9a1c6aec104d24eea(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:41,606 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:41,606 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/A, priority=13, startTime=1733193221444; duration=0sec 2024-12-03T02:33:41,607 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:41,607 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:A 2024-12-03T02:33:41,607 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:41,622 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:41,622 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/C is initiating minor compaction (all files) 2024-12-03T02:33:41,622 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/C in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:41,622 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/d86d302a5f4d486f91ceaf9de6f27eb3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/6da36179f470417087f729bfa44b9ba4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e48681d42e6c44159c5e06d8e2e8dc56] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=36.4 K 2024-12-03T02:33:41,623 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting d86d302a5f4d486f91ceaf9de6f27eb3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733193218320 2024-12-03T02:33:41,624 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da36179f470417087f729bfa44b9ba4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733193218676 2024-12-03T02:33:41,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193281615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193281615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,629 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e48681d42e6c44159c5e06d8e2e8dc56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733193220053 2024-12-03T02:33:41,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193281615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193281626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193281627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,641 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#C#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:41,642 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/e25f891796a54e3ba8cbaf9d4a61629c is 50, key is test_row_0/C:col10/1733193221239/Put/seqid=0 2024-12-03T02:33:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741886_1062 (size=12813) 2024-12-03T02:33:41,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193281835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193281835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193281835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193281854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193281855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:41,928 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/b385de93a41c409981c8cb6bea47b794 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/b385de93a41c409981c8cb6bea47b794 2024-12-03T02:33:41,938 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/B of a2199fc0aeeef0203e54ad44ea3ac1a7 into b385de93a41c409981c8cb6bea47b794(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:41,938 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:41,938 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/B, priority=13, startTime=1733193221446; duration=0sec 2024-12-03T02:33:41,938 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:41,938 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:B 2024-12-03T02:33:41,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/25f8be2a822d4c2bad8bda1b8e9eba42 2024-12-03T02:33:42,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/ec36fbfc565a41e5aa6971acc2db1628 is 50, key is test_row_0/C:col10/1733193221446/Put/seqid=0 2024-12-03T02:33:42,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741887_1063 (size=12301) 2024-12-03T02:33:42,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/ec36fbfc565a41e5aa6971acc2db1628 2024-12-03T02:33:42,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/e85d1ba9797c4d02bd97e0ba5f91155d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e85d1ba9797c4d02bd97e0ba5f91155d 2024-12-03T02:33:42,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e85d1ba9797c4d02bd97e0ba5f91155d, entries=200, sequenceid=439, filesize=14.4 K 2024-12-03T02:33:42,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/25f8be2a822d4c2bad8bda1b8e9eba42 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/25f8be2a822d4c2bad8bda1b8e9eba42 2024-12-03T02:33:42,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/25f8be2a822d4c2bad8bda1b8e9eba42, entries=150, sequenceid=439, filesize=12.0 K 2024-12-03T02:33:42,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/ec36fbfc565a41e5aa6971acc2db1628 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ec36fbfc565a41e5aa6971acc2db1628 2024-12-03T02:33:42,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ec36fbfc565a41e5aa6971acc2db1628, entries=150, sequenceid=439, filesize=12.0 K 2024-12-03T02:33:42,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~199.06 KB/203840, currentSize=107.34 KB/109920 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 641ms, sequenceid=439, compaction requested=false 2024-12-03T02:33:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:42,124 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/e25f891796a54e3ba8cbaf9d4a61629c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e25f891796a54e3ba8cbaf9d4a61629c 2024-12-03T02:33:42,141 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/C of a2199fc0aeeef0203e54ad44ea3ac1a7 into e25f891796a54e3ba8cbaf9d4a61629c(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:42,141 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:42,141 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/C, priority=13, startTime=1733193221446; duration=0sec 2024-12-03T02:33:42,141 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:42,141 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:C 2024-12-03T02:33:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:42,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T02:33:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:42,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/a07afe4316844d2ca2cd94c2087ac402 is 50, key is test_row_0/A:col10/1733193221496/Put/seqid=0 2024-12-03T02:33:42,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741888_1064 (size=14741) 2024-12-03T02:33:42,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/a07afe4316844d2ca2cd94c2087ac402 2024-12-03T02:33:42,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193282171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193282171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193282179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193282179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193282175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/0b0841602fc9462daf5c4d40d10bbf5c is 50, key is test_row_0/B:col10/1733193221496/Put/seqid=0 2024-12-03T02:33:42,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741889_1065 (size=12301) 2024-12-03T02:33:42,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193282295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193282297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193282295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193282308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193282308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193282504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193282504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193282515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193282515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193282524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/0b0841602fc9462daf5c4d40d10bbf5c 2024-12-03T02:33:42,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/2fb4bc820adb43a49c5a4f8c7c6610f1 is 50, key is test_row_0/C:col10/1733193221496/Put/seqid=0 2024-12-03T02:33:42,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741890_1066 (size=12301) 2024-12-03T02:33:42,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/2fb4bc820adb43a49c5a4f8c7c6610f1 2024-12-03T02:33:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/a07afe4316844d2ca2cd94c2087ac402 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/a07afe4316844d2ca2cd94c2087ac402 2024-12-03T02:33:42,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/a07afe4316844d2ca2cd94c2087ac402, entries=200, sequenceid=462, filesize=14.4 K 2024-12-03T02:33:42,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/0b0841602fc9462daf5c4d40d10bbf5c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0b0841602fc9462daf5c4d40d10bbf5c 2024-12-03T02:33:42,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0b0841602fc9462daf5c4d40d10bbf5c, entries=150, sequenceid=462, filesize=12.0 K 2024-12-03T02:33:42,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/2fb4bc820adb43a49c5a4f8c7c6610f1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/2fb4bc820adb43a49c5a4f8c7c6610f1 2024-12-03T02:33:42,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/2fb4bc820adb43a49c5a4f8c7c6610f1, entries=150, sequenceid=462, filesize=12.0 K 2024-12-03T02:33:42,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 596ms, sequenceid=462, compaction requested=true 2024-12-03T02:33:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:42,743 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:42,743 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:42,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:42,749 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:42,749 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42295 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:42,749 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/B is initiating minor compaction (all files) 2024-12-03T02:33:42,749 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/A is initiating minor compaction (all files) 2024-12-03T02:33:42,749 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/A in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:42,749 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/4f94ed55141648c9a1c6aec104d24eea, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e85d1ba9797c4d02bd97e0ba5f91155d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/a07afe4316844d2ca2cd94c2087ac402] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=41.3 K 2024-12-03T02:33:42,750 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/B in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:42,750 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/b385de93a41c409981c8cb6bea47b794, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/25f8be2a822d4c2bad8bda1b8e9eba42, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0b0841602fc9462daf5c4d40d10bbf5c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=36.5 K 2024-12-03T02:33:42,750 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f94ed55141648c9a1c6aec104d24eea, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733193220053 2024-12-03T02:33:42,753 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e85d1ba9797c4d02bd97e0ba5f91155d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733193221319 2024-12-03T02:33:42,755 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a07afe4316844d2ca2cd94c2087ac402, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733193221483 2024-12-03T02:33:42,758 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b385de93a41c409981c8cb6bea47b794, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733193220053 2024-12-03T02:33:42,761 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f8be2a822d4c2bad8bda1b8e9eba42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733193221331 2024-12-03T02:33:42,762 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b0841602fc9462daf5c4d40d10bbf5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733193221483 2024-12-03T02:33:42,794 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#B#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:42,794 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/3b7406a8474040b6b05a33a82461e51e is 50, key is test_row_0/B:col10/1733193221496/Put/seqid=0 2024-12-03T02:33:42,798 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#A#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:42,799 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/5dd3f506eda94ea68194356a7cde90c6 is 50, key is test_row_0/A:col10/1733193221496/Put/seqid=0 2024-12-03T02:33:42,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:42,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T02:33:42,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:42,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:33:42,836 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:42,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:33:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-03T02:33:42,840 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:33:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T02:33:42,842 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:33:42,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:33:42,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/84a99241b1d04f67a9078de612b0ed60 is 50, key is test_row_0/A:col10/1733193222174/Put/seqid=0 2024-12-03T02:33:42,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741892_1068 (size=12915) 2024-12-03T02:33:42,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741891_1067 (size=12915) 2024-12-03T02:33:42,860 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/3b7406a8474040b6b05a33a82461e51e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/3b7406a8474040b6b05a33a82461e51e 2024-12-03T02:33:42,862 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/5dd3f506eda94ea68194356a7cde90c6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/5dd3f506eda94ea68194356a7cde90c6 2024-12-03T02:33:42,873 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/B of a2199fc0aeeef0203e54ad44ea3ac1a7 into 3b7406a8474040b6b05a33a82461e51e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:42,873 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:42,873 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/B, priority=13, startTime=1733193222743; duration=0sec 2024-12-03T02:33:42,873 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:42,873 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:B 2024-12-03T02:33:42,873 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:42,879 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/A of a2199fc0aeeef0203e54ad44ea3ac1a7 into 5dd3f506eda94ea68194356a7cde90c6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:42,879 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:42,879 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/A, priority=13, startTime=1733193222743; duration=0sec 2024-12-03T02:33:42,880 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:42,880 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:A 2024-12-03T02:33:42,881 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:42,881 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/C is initiating minor compaction (all files) 2024-12-03T02:33:42,881 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/C in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:42,882 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e25f891796a54e3ba8cbaf9d4a61629c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ec36fbfc565a41e5aa6971acc2db1628, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/2fb4bc820adb43a49c5a4f8c7c6610f1] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=36.5 K 2024-12-03T02:33:42,883 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e25f891796a54e3ba8cbaf9d4a61629c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733193220053 2024-12-03T02:33:42,884 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ec36fbfc565a41e5aa6971acc2db1628, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733193221331 2024-12-03T02:33:42,885 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb4bc820adb43a49c5a4f8c7c6610f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733193221483 2024-12-03T02:33:42,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741893_1069 (size=14741) 2024-12-03T02:33:42,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/84a99241b1d04f67a9078de612b0ed60 2024-12-03T02:33:42,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733193222870, max timestamp=1733193222898Num uniques 859; 2024-12-03T02:33:42,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:42,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193222872, max timestamp=1733193222898Num uniques 859; 2024-12-03T02:33:42,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:33:42,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193222872, max timestamp=1733193222898Num uniques 859; 2024-12-03T02:33:42,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:33:42,922 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:42,923 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/ecd7c706f16849ebb86ca99c513a9500 is 50, key is test_row_0/C:col10/1733193221496/Put/seqid=0 2024-12-03T02:33:42,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/679e88f6e4404265a8bb1269d42f1422 is 50, key is test_row_0/B:col10/1733193222174/Put/seqid=0 2024-12-03T02:33:42,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193282934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193282934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T02:33:42,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193282938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741895_1071 (size=12301) 2024-12-03T02:33:42,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193282942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193282942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:42,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/679e88f6e4404265a8bb1269d42f1422 2024-12-03T02:33:42,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741894_1070 (size=12915) 2024-12-03T02:33:42,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/800fd19d253f40c5b3683b022dae532d is 50, key is test_row_0/C:col10/1733193222174/Put/seqid=0 2024-12-03T02:33:42,977 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/ecd7c706f16849ebb86ca99c513a9500 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ecd7c706f16849ebb86ca99c513a9500 2024-12-03T02:33:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741896_1072 (size=12301) 2024-12-03T02:33:42,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/800fd19d253f40c5b3683b022dae532d 2024-12-03T02:33:42,994 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/C of a2199fc0aeeef0203e54ad44ea3ac1a7 into ecd7c706f16849ebb86ca99c513a9500(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:42,994 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:42,994 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/C, priority=13, startTime=1733193222743; duration=0sec 2024-12-03T02:33:42,994 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:42,994 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:C 2024-12-03T02:33:42,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T02:33:42,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:42,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:42,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:42,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:42,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/84a99241b1d04f67a9078de612b0ed60 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/84a99241b1d04f67a9078de612b0ed60 2024-12-03T02:33:42,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/84a99241b1d04f67a9078de612b0ed60, entries=200, sequenceid=475, filesize=14.4 K 2024-12-03T02:33:43,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/679e88f6e4404265a8bb1269d42f1422 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/679e88f6e4404265a8bb1269d42f1422 2024-12-03T02:33:43,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/679e88f6e4404265a8bb1269d42f1422, entries=150, sequenceid=475, filesize=12.0 K 2024-12-03T02:33:43,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/800fd19d253f40c5b3683b022dae532d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/800fd19d253f40c5b3683b022dae532d 2024-12-03T02:33:43,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/800fd19d253f40c5b3683b022dae532d, entries=150, sequenceid=475, filesize=12.0 K 2024-12-03T02:33:43,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=123.00 KB/125950 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 206ms, sequenceid=475, compaction requested=false 2024-12-03T02:33:43,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:43,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-12-03T02:33:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:43,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/c866325165c14955b59c8885fb6cf150 is 50, key is test_row_0/A:col10/1733193223047/Put/seqid=0 2024-12-03T02:33:43,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193283068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193283076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741897_1073 (size=12301) 2024-12-03T02:33:43,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193283079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/c866325165c14955b59c8885fb6cf150 2024-12-03T02:33:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193283080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193283081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/e53708cca0c340659c27bd74b99cecd7 is 50, key is test_row_0/B:col10/1733193223047/Put/seqid=0 2024-12-03T02:33:43,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741898_1074 (size=12301) 2024-12-03T02:33:43,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T02:33:43,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:43,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T02:33:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193283184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193283195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193283195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193283195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193283195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T02:33:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193283394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193283404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193283405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193283405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193283411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T02:33:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T02:33:43,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/e53708cca0c340659c27bd74b99cecd7 2024-12-03T02:33:43,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/cfba5ca36aa54c41bf7daca2aa891439 is 50, key is test_row_0/C:col10/1733193223047/Put/seqid=0 2024-12-03T02:33:43,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741899_1075 (size=12301) 2024-12-03T02:33:43,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T02:33:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:43,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193283705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193283715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193283715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193283715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193283725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:43,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T02:33:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T02:33:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:43,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T02:33:43,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/cfba5ca36aa54c41bf7daca2aa891439 2024-12-03T02:33:44,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/c866325165c14955b59c8885fb6cf150 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c866325165c14955b59c8885fb6cf150 2024-12-03T02:33:44,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c866325165c14955b59c8885fb6cf150, entries=150, sequenceid=513, filesize=12.0 K 2024-12-03T02:33:44,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/e53708cca0c340659c27bd74b99cecd7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e53708cca0c340659c27bd74b99cecd7 2024-12-03T02:33:44,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e53708cca0c340659c27bd74b99cecd7, entries=150, sequenceid=513, filesize=12.0 K 2024-12-03T02:33:44,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/cfba5ca36aa54c41bf7daca2aa891439 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/cfba5ca36aa54c41bf7daca2aa891439 2024-12-03T02:33:44,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/cfba5ca36aa54c41bf7daca2aa891439, entries=150, sequenceid=513, filesize=12.0 K 2024-12-03T02:33:44,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=60.38 KB/61830 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 992ms, sequenceid=513, compaction requested=true 2024-12-03T02:33:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:44,041 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:44,041 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:44,042 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:44,042 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:44,042 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/A is initiating minor compaction (all files) 2024-12-03T02:33:44,042 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/B is initiating minor compaction (all files) 2024-12-03T02:33:44,042 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/A in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:44,042 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/B in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:44,043 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/3b7406a8474040b6b05a33a82461e51e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/679e88f6e4404265a8bb1269d42f1422, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e53708cca0c340659c27bd74b99cecd7] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=36.6 K 2024-12-03T02:33:44,043 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/5dd3f506eda94ea68194356a7cde90c6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/84a99241b1d04f67a9078de612b0ed60, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c866325165c14955b59c8885fb6cf150] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=39.0 K 2024-12-03T02:33:44,043 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd3f506eda94ea68194356a7cde90c6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733193221483 2024-12-03T02:33:44,043 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7406a8474040b6b05a33a82461e51e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733193221483 2024-12-03T02:33:44,043 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84a99241b1d04f67a9078de612b0ed60, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733193222162 2024-12-03T02:33:44,043 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 679e88f6e4404265a8bb1269d42f1422, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733193222174 2024-12-03T02:33:44,044 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting c866325165c14955b59c8885fb6cf150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733193222931 2024-12-03T02:33:44,044 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e53708cca0c340659c27bd74b99cecd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733193222931 2024-12-03T02:33:44,054 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#B#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:44,055 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/262d808343ed45229be225d26b391524 is 50, key is test_row_0/B:col10/1733193223047/Put/seqid=0 2024-12-03T02:33:44,057 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:44,057 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/64ed31e7584a4a19846ef644774a4e3e is 50, key is test_row_0/A:col10/1733193223047/Put/seqid=0 2024-12-03T02:33:44,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741901_1077 (size=13017) 2024-12-03T02:33:44,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741900_1076 (size=13017) 2024-12-03T02:33:44,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T02:33:44,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:44,083 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T02:33:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/3ff79598a6ed4e0daa5293627cc90ad0 is 50, key is test_row_0/A:col10/1733193223077/Put/seqid=0 2024-12-03T02:33:44,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741902_1078 (size=12301) 2024-12-03T02:33:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:44,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193284243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193284243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193284247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733193224234, max timestamp=1733193224248Num uniques 859; 2024-12-03T02:33:44,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733193224234, max timestamp=1733193224248Num uniques 859; 2024-12-03T02:33:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:44,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:44,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733193224234, max timestamp=1733193224248Num uniques 859; 2024-12-03T02:33:44,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193284304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193284309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193284364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193284364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193284365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193284414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193284424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,478 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/262d808343ed45229be225d26b391524 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/262d808343ed45229be225d26b391524 2024-12-03T02:33:44,480 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/64ed31e7584a4a19846ef644774a4e3e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/64ed31e7584a4a19846ef644774a4e3e 2024-12-03T02:33:44,490 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/B of a2199fc0aeeef0203e54ad44ea3ac1a7 into 262d808343ed45229be225d26b391524(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:44,490 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:44,490 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/B, priority=13, startTime=1733193224041; duration=0sec 2024-12-03T02:33:44,491 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/A of a2199fc0aeeef0203e54ad44ea3ac1a7 into 64ed31e7584a4a19846ef644774a4e3e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:44,491 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:44,491 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/A, priority=13, startTime=1733193224041; duration=0sec 2024-12-03T02:33:44,491 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:44,491 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:A 2024-12-03T02:33:44,491 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:44,492 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:44,492 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:B 2024-12-03T02:33:44,493 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:44,494 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/C is initiating minor compaction (all files) 2024-12-03T02:33:44,494 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/C in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:44,494 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ecd7c706f16849ebb86ca99c513a9500, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/800fd19d253f40c5b3683b022dae532d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/cfba5ca36aa54c41bf7daca2aa891439] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=36.6 K 2024-12-03T02:33:44,495 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecd7c706f16849ebb86ca99c513a9500, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733193221483 2024-12-03T02:33:44,495 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 800fd19d253f40c5b3683b022dae532d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1733193222174 2024-12-03T02:33:44,496 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfba5ca36aa54c41bf7daca2aa891439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733193222931 2024-12-03T02:33:44,506 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:44,507 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/7259e890654f4201b7871eeac9495f02 is 50, key is test_row_0/C:col10/1733193223047/Put/seqid=0 2024-12-03T02:33:44,509 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/3ff79598a6ed4e0daa5293627cc90ad0 2024-12-03T02:33:44,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741903_1079 (size=13017) 2024-12-03T02:33:44,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/673d7ae2362d4cd4b4c36efa6c93a266 is 50, key is test_row_0/B:col10/1733193223077/Put/seqid=0 2024-12-03T02:33:44,526 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/7259e890654f4201b7871eeac9495f02 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/7259e890654f4201b7871eeac9495f02 2024-12-03T02:33:44,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741904_1080 (size=12301) 2024-12-03T02:33:44,531 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/673d7ae2362d4cd4b4c36efa6c93a266 2024-12-03T02:33:44,536 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/C of a2199fc0aeeef0203e54ad44ea3ac1a7 into 7259e890654f4201b7871eeac9495f02(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:44,536 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:44,536 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/C, priority=13, startTime=1733193224041; duration=0sec 2024-12-03T02:33:44,536 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:44,536 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:C 2024-12-03T02:33:44,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/839e98ebcb7c41af9b1eee8d4703f71e is 50, key is test_row_0/C:col10/1733193223077/Put/seqid=0 2024-12-03T02:33:44,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741905_1081 (size=12301) 2024-12-03T02:33:44,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193284575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193284576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193284576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193284624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193284634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193284885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193284885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193284885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193284944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193284945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:44,951 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/839e98ebcb7c41af9b1eee8d4703f71e 2024-12-03T02:33:44,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/3ff79598a6ed4e0daa5293627cc90ad0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3ff79598a6ed4e0daa5293627cc90ad0 2024-12-03T02:33:44,963 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3ff79598a6ed4e0daa5293627cc90ad0, entries=150, sequenceid=525, filesize=12.0 K 2024-12-03T02:33:44,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/673d7ae2362d4cd4b4c36efa6c93a266 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/673d7ae2362d4cd4b4c36efa6c93a266 2024-12-03T02:33:44,969 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/673d7ae2362d4cd4b4c36efa6c93a266, entries=150, sequenceid=525, filesize=12.0 K 2024-12-03T02:33:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/839e98ebcb7c41af9b1eee8d4703f71e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/839e98ebcb7c41af9b1eee8d4703f71e 2024-12-03T02:33:44,977 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/839e98ebcb7c41af9b1eee8d4703f71e, entries=150, sequenceid=525, filesize=12.0 K 2024-12-03T02:33:44,978 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 896ms, sequenceid=525, compaction requested=false 2024-12-03T02:33:44,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:44,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:44,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-03T02:33:44,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-03T02:33:44,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-03T02:33:44,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1380 sec 2024-12-03T02:33:44,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.1450 sec 2024-12-03T02:33:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T02:33:44,985 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:44,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:33:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-03T02:33:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T02:33:44,987 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:33:44,987 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:33:44,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:33:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T02:33:45,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T02:33:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:45,142 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-03T02:33:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:45,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:45,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/47aec078898946d7b6e8169c01332d12 is 50, key is test_row_0/A:col10/1733193224303/Put/seqid=0 2024-12-03T02:33:45,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741906_1082 (size=12301) 2024-12-03T02:33:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T02:33:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:45,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:45,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193285404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193285411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193285415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193285455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193285455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193285525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193285527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193285527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,552 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/47aec078898946d7b6e8169c01332d12 2024-12-03T02:33:45,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/546776c771b3450585632507458ec107 is 50, key is test_row_0/B:col10/1733193224303/Put/seqid=0 2024-12-03T02:33:45,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741907_1083 (size=12301) 2024-12-03T02:33:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T02:33:45,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193285735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193285735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193285736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:45,980 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/546776c771b3450585632507458ec107 2024-12-03T02:33:46,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/0a412b8c92194dd3b781560f9c50cf34 is 50, key is test_row_0/C:col10/1733193224303/Put/seqid=0 2024-12-03T02:33:46,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193286051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193286058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193286058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741908_1084 (size=12301) 2024-12-03T02:33:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T02:33:46,470 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/0a412b8c92194dd3b781560f9c50cf34 2024-12-03T02:33:46,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/47aec078898946d7b6e8169c01332d12 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/47aec078898946d7b6e8169c01332d12 2024-12-03T02:33:46,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193286475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193286476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,483 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in a2199fc0aeeef0203e54ad44ea3ac1a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:33:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,484 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in a2199fc0aeeef0203e54ad44ea3ac1a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:33:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,484 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/47aec078898946d7b6e8169c01332d12, entries=150, sequenceid=562, filesize=12.0 K 2024-12-03T02:33:46,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/546776c771b3450585632507458ec107 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/546776c771b3450585632507458ec107 2024-12-03T02:33:46,505 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/546776c771b3450585632507458ec107, entries=150, sequenceid=562, filesize=12.0 K 2024-12-03T02:33:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/0a412b8c92194dd3b781560f9c50cf34 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/0a412b8c92194dd3b781560f9c50cf34 2024-12-03T02:33:46,521 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/0a412b8c92194dd3b781560f9c50cf34, entries=150, sequenceid=562, filesize=12.0 K 2024-12-03T02:33:46,523 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 1380ms, sequenceid=562, compaction requested=true 2024-12-03T02:33:46,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:46,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:46,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-03T02:33:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-03T02:33:46,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T02:33:46,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5390 sec 2024-12-03T02:33:46,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5450 sec 2024-12-03T02:33:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:46,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T02:33:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:46,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/14632105951743d89eb32edb8c400045 is 50, key is test_row_0/A:col10/1733193225401/Put/seqid=0 2024-12-03T02:33:46,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741909_1085 (size=14741) 2024-12-03T02:33:46,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733193226588, max timestamp=1733193226607Num uniques 859; 2024-12-03T02:33:46,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T02:33:46,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733193226588, max timestamp=1733193226607Num uniques 859; 2024-12-03T02:33:46,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T02:33:46,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733193226588, max timestamp=1733193226607Num uniques 859; 2024-12-03T02:33:46,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T02:33:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193286629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193286633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193286638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193286748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193286752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193286755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193286972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193286972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193286972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:46,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/14632105951743d89eb32edb8c400045 2024-12-03T02:33:47,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/96d1d52f66474753b695f5f04a83c934 is 50, key is test_row_0/B:col10/1733193225401/Put/seqid=0 2024-12-03T02:33:47,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741910_1086 (size=12301) 2024-12-03T02:33:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T02:33:47,136 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:47,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:33:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-03T02:33:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T02:33:47,154 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:33:47,156 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:33:47,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:33:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T02:33:47,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193287285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:47,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193287285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193287285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:47,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T02:33:47,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:47,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:47,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:47,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:47,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:47,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/96d1d52f66474753b695f5f04a83c934 2024-12-03T02:33:47,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/3e4ac9ef0e664cf0b71c1e366c6e9752 is 50, key is test_row_0/C:col10/1733193225401/Put/seqid=0 2024-12-03T02:33:47,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741911_1087 (size=12301) 2024-12-03T02:33:47,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/3e4ac9ef0e664cf0b71c1e366c6e9752 2024-12-03T02:33:47,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/14632105951743d89eb32edb8c400045 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/14632105951743d89eb32edb8c400045 2024-12-03T02:33:47,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/14632105951743d89eb32edb8c400045, entries=200, sequenceid=574, filesize=14.4 K 2024-12-03T02:33:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/96d1d52f66474753b695f5f04a83c934 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96d1d52f66474753b695f5f04a83c934 2024-12-03T02:33:47,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T02:33:47,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:47,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:47,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:47,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:47,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T02:33:47,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96d1d52f66474753b695f5f04a83c934, entries=150, sequenceid=574, filesize=12.0 K 2024-12-03T02:33:47,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/3e4ac9ef0e664cf0b71c1e366c6e9752 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3e4ac9ef0e664cf0b71c1e366c6e9752 2024-12-03T02:33:47,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3e4ac9ef0e664cf0b71c1e366c6e9752, entries=150, sequenceid=574, filesize=12.0 K 2024-12-03T02:33:47,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 917ms, sequenceid=574, compaction requested=true 2024-12-03T02:33:47,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:47,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:47,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:47,484 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:47,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:47,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:47,484 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:47,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:47,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:47,486 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:47,486 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:47,486 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/B is initiating minor compaction (all files) 2024-12-03T02:33:47,486 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/A is initiating minor compaction (all files) 2024-12-03T02:33:47,486 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/B in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:47,486 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/A in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:47,486 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/262d808343ed45229be225d26b391524, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/673d7ae2362d4cd4b4c36efa6c93a266, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/546776c771b3450585632507458ec107, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96d1d52f66474753b695f5f04a83c934] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=48.8 K 2024-12-03T02:33:47,486 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/64ed31e7584a4a19846ef644774a4e3e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3ff79598a6ed4e0daa5293627cc90ad0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/47aec078898946d7b6e8169c01332d12, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/14632105951743d89eb32edb8c400045] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=51.1 K 2024-12-03T02:33:47,487 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 262d808343ed45229be225d26b391524, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733193222931 2024-12-03T02:33:47,487 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ed31e7584a4a19846ef644774a4e3e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733193222931 2024-12-03T02:33:47,488 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ff79598a6ed4e0daa5293627cc90ad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733193223076 2024-12-03T02:33:47,488 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 673d7ae2362d4cd4b4c36efa6c93a266, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733193223076 2024-12-03T02:33:47,488 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47aec078898946d7b6e8169c01332d12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733193224300 2024-12-03T02:33:47,489 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 546776c771b3450585632507458ec107, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733193224300 2024-12-03T02:33:47,489 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14632105951743d89eb32edb8c400045, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733193225401 2024-12-03T02:33:47,489 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 96d1d52f66474753b695f5f04a83c934, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733193225401 2024-12-03T02:33:47,505 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:47,506 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/8f87eedd1ae345dca48fb65f764660bf is 50, key is test_row_0/A:col10/1733193225401/Put/seqid=0 2024-12-03T02:33:47,514 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:47,514 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/802c76f6311b4163b92e66171ad662c5 is 50, key is test_row_0/B:col10/1733193225401/Put/seqid=0 2024-12-03T02:33:47,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741912_1088 (size=13153) 2024-12-03T02:33:47,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741913_1089 (size=13153) 2024-12-03T02:33:47,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T02:33:47,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:47,616 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=127.47 KB heapSize=351 KB 2024-12-03T02:33:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:47,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:47,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/569856f0cfb94e749a05cf58a309fb36 is 50, key is test_row_0/A:col10/1733193226627/Put/seqid=0 2024-12-03T02:33:47,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741914_1090 (size=12301) 2024-12-03T02:33:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T02:33:47,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. as already flushing 2024-12-03T02:33:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:47,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193287821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193287822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193287823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:47,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193287926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:47,931 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/8f87eedd1ae345dca48fb65f764660bf as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8f87eedd1ae345dca48fb65f764660bf 2024-12-03T02:33:47,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193287934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:47,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193287934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:47,946 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/A of a2199fc0aeeef0203e54ad44ea3ac1a7 into 8f87eedd1ae345dca48fb65f764660bf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:47,946 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:47,946 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/A, priority=12, startTime=1733193227484; duration=0sec 2024-12-03T02:33:47,946 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:47,946 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:A 2024-12-03T02:33:47,946 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:33:47,946 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/802c76f6311b4163b92e66171ad662c5 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/802c76f6311b4163b92e66171ad662c5 2024-12-03T02:33:47,952 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:33:47,953 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/C is initiating minor compaction (all files) 2024-12-03T02:33:47,953 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/C in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:47,953 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/7259e890654f4201b7871eeac9495f02, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/839e98ebcb7c41af9b1eee8d4703f71e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/0a412b8c92194dd3b781560f9c50cf34, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3e4ac9ef0e664cf0b71c1e366c6e9752] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=48.8 K 2024-12-03T02:33:47,954 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7259e890654f4201b7871eeac9495f02, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733193222931 2024-12-03T02:33:47,955 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 839e98ebcb7c41af9b1eee8d4703f71e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733193223076 2024-12-03T02:33:47,957 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a412b8c92194dd3b781560f9c50cf34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733193224300 2024-12-03T02:33:47,957 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e4ac9ef0e664cf0b71c1e366c6e9752, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733193225401 2024-12-03T02:33:47,959 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/B of a2199fc0aeeef0203e54ad44ea3ac1a7 into 802c76f6311b4163b92e66171ad662c5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:47,959 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:47,959 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/B, priority=12, startTime=1733193227484; duration=0sec 2024-12-03T02:33:47,960 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:47,960 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:B 2024-12-03T02:33:47,973 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:47,973 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/3303b10a8d5f441e8b5c7e9b0527f9a0 is 50, key is test_row_0/C:col10/1733193225401/Put/seqid=0 2024-12-03T02:33:47,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741915_1091 (size=13153) 2024-12-03T02:33:48,013 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/3303b10a8d5f441e8b5c7e9b0527f9a0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3303b10a8d5f441e8b5c7e9b0527f9a0 2024-12-03T02:33:48,027 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/C of a2199fc0aeeef0203e54ad44ea3ac1a7 into 3303b10a8d5f441e8b5c7e9b0527f9a0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:48,027 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:48,027 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/C, priority=12, startTime=1733193227484; duration=0sec 2024-12-03T02:33:48,029 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:48,029 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:C 2024-12-03T02:33:48,032 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/569856f0cfb94e749a05cf58a309fb36 2024-12-03T02:33:48,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/79b9fb5c13894267b25152cebb81051a is 50, key is test_row_0/B:col10/1733193226627/Put/seqid=0 2024-12-03T02:33:48,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741916_1092 (size=12301) 2024-12-03T02:33:48,045 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/79b9fb5c13894267b25152cebb81051a 2024-12-03T02:33:48,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/630d75c574384dffa7a9c52ba7bfedad is 50, key is test_row_0/C:col10/1733193226627/Put/seqid=0 2024-12-03T02:33:48,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741917_1093 (size=12301) 2024-12-03T02:33:48,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193288134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193288145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193288145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T02:33:48,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193288445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193288454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193288454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,466 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/630d75c574384dffa7a9c52ba7bfedad 2024-12-03T02:33:48,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/569856f0cfb94e749a05cf58a309fb36 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/569856f0cfb94e749a05cf58a309fb36 2024-12-03T02:33:48,479 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/569856f0cfb94e749a05cf58a309fb36, entries=150, sequenceid=607, filesize=12.0 K 2024-12-03T02:33:48,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/79b9fb5c13894267b25152cebb81051a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/79b9fb5c13894267b25152cebb81051a 2024-12-03T02:33:48,484 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/79b9fb5c13894267b25152cebb81051a, entries=150, sequenceid=607, filesize=12.0 K 2024-12-03T02:33:48,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/630d75c574384dffa7a9c52ba7bfedad as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/630d75c574384dffa7a9c52ba7bfedad 2024-12-03T02:33:48,491 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/630d75c574384dffa7a9c52ba7bfedad, entries=150, sequenceid=607, filesize=12.0 K 2024-12-03T02:33:48,492 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~350.95 KB/359376, currentSize=67.09 KB/68700 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 876ms, sequenceid=607, compaction requested=false 2024-12-03T02:33:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:48,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-03T02:33:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-03T02:33:48,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T02:33:48,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3380 sec 2024-12-03T02:33:48,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3570 sec 2024-12-03T02:33:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:48,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T02:33:48,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:48,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:48,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:48,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:48,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:48,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:48,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/7afe53fab0de442d81b42ddde57e6b7b is 50, key is test_row_0/A:col10/1733193227819/Put/seqid=0 2024-12-03T02:33:48,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741918_1094 (size=14741) 2024-12-03T02:33:48,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/7afe53fab0de442d81b42ddde57e6b7b 2024-12-03T02:33:48,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/c85b6de0c69b42b19ab0c1a8c93dc0d9 is 50, key is test_row_0/B:col10/1733193227819/Put/seqid=0 2024-12-03T02:33:48,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741919_1095 (size=12301) 2024-12-03T02:33:48,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/c85b6de0c69b42b19ab0c1a8c93dc0d9 2024-12-03T02:33:48,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/051ec3165e95434eb7686c61f00828bc is 50, key is test_row_0/C:col10/1733193227819/Put/seqid=0 2024-12-03T02:33:48,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741920_1096 (size=12301) 2024-12-03T02:33:48,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=624 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/051ec3165e95434eb7686c61f00828bc 2024-12-03T02:33:48,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193288554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193288554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/7afe53fab0de442d81b42ddde57e6b7b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7afe53fab0de442d81b42ddde57e6b7b 2024-12-03T02:33:48,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7afe53fab0de442d81b42ddde57e6b7b, entries=200, sequenceid=624, filesize=14.4 K 2024-12-03T02:33:48,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/c85b6de0c69b42b19ab0c1a8c93dc0d9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/c85b6de0c69b42b19ab0c1a8c93dc0d9 2024-12-03T02:33:48,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/c85b6de0c69b42b19ab0c1a8c93dc0d9, entries=150, sequenceid=624, filesize=12.0 K 2024-12-03T02:33:48,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/051ec3165e95434eb7686c61f00828bc as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/051ec3165e95434eb7686c61f00828bc 2024-12-03T02:33:48,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/051ec3165e95434eb7686c61f00828bc, entries=150, sequenceid=624, filesize=12.0 K 2024-12-03T02:33:48,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 104ms, sequenceid=624, compaction requested=true 2024-12-03T02:33:48,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:48,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:48,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:48,611 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:48,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:48,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:48,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:48,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:48,611 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:48,615 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:48,615 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/B is initiating minor compaction (all files) 2024-12-03T02:33:48,615 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/B in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:48,615 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/802c76f6311b4163b92e66171ad662c5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/79b9fb5c13894267b25152cebb81051a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/c85b6de0c69b42b19ab0c1a8c93dc0d9] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=36.9 K 2024-12-03T02:33:48,616 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 802c76f6311b4163b92e66171ad662c5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733193225401 2024-12-03T02:33:48,617 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:48,617 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/A is initiating minor compaction (all files) 2024-12-03T02:33:48,617 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b9fb5c13894267b25152cebb81051a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1733193226627 2024-12-03T02:33:48,617 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/A in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:48,617 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8f87eedd1ae345dca48fb65f764660bf, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/569856f0cfb94e749a05cf58a309fb36, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7afe53fab0de442d81b42ddde57e6b7b] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=39.3 K 2024-12-03T02:33:48,618 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting c85b6de0c69b42b19ab0c1a8c93dc0d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1733193227819 2024-12-03T02:33:48,620 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f87eedd1ae345dca48fb65f764660bf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733193225401 2024-12-03T02:33:48,621 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 569856f0cfb94e749a05cf58a309fb36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1733193226627 2024-12-03T02:33:48,623 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7afe53fab0de442d81b42ddde57e6b7b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1733193227802 2024-12-03T02:33:48,646 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:48,646 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/19cfe2f35f1c43a1ac3319ab2e6ed365 is 50, key is test_row_0/B:col10/1733193227819/Put/seqid=0 2024-12-03T02:33:48,654 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#A#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:48,654 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/591675012b98496f8c0026085167bbf3 is 50, key is test_row_0/A:col10/1733193227819/Put/seqid=0 2024-12-03T02:33:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:48,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T02:33:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:48,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:48,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741921_1097 (size=13255) 2024-12-03T02:33:48,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741922_1098 (size=13255) 2024-12-03T02:33:48,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/e0b4a4a7d4494d9b96e66a18a3bf6a3e is 50, key is test_row_0/A:col10/1733193228544/Put/seqid=0 2024-12-03T02:33:48,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741923_1099 (size=14741) 2024-12-03T02:33:48,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193288703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193288715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193288825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193288825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48094 deadline: 1733193288954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48130 deadline: 1733193288965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48064 deadline: 1733193288975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:49,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:49,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48102 deadline: 1733193289034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48062 deadline: 1733193289034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:33:49,104 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/19cfe2f35f1c43a1ac3319ab2e6ed365 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/19cfe2f35f1c43a1ac3319ab2e6ed365 2024-12-03T02:33:49,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/e0b4a4a7d4494d9b96e66a18a3bf6a3e 2024-12-03T02:33:49,125 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/591675012b98496f8c0026085167bbf3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/591675012b98496f8c0026085167bbf3 2024-12-03T02:33:49,125 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-03T02:33:49,125 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:33:49,126 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-03T02:33:49,126 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:33:49,128 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-03T02:33:49,128 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:33:49,134 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-03T02:33:49,134 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:33:49,136 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,136 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,136 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,137 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,137 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,137 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,138 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,139 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,142 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/B of a2199fc0aeeef0203e54ad44ea3ac1a7 into 19cfe2f35f1c43a1ac3319ab2e6ed365(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:49,142 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:49,142 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/B, priority=13, startTime=1733193228611; duration=0sec 2024-12-03T02:33:49,143 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:49,143 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:B 2024-12-03T02:33:49,143 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:49,144 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/A of a2199fc0aeeef0203e54ad44ea3ac1a7 into 591675012b98496f8c0026085167bbf3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:49,144 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:49,144 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/A, priority=13, startTime=1733193228611; duration=0sec 2024-12-03T02:33:49,145 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:49,145 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:A 2024-12-03T02:33:49,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/96059f17688c44bd9ef637a06b5c65db is 50, key is test_row_0/B:col10/1733193228544/Put/seqid=0 2024-12-03T02:33:49,149 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:49,149 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): a2199fc0aeeef0203e54ad44ea3ac1a7/C is initiating minor compaction (all files) 2024-12-03T02:33:49,149 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2199fc0aeeef0203e54ad44ea3ac1a7/C in TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:49,149 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3303b10a8d5f441e8b5c7e9b0527f9a0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/630d75c574384dffa7a9c52ba7bfedad, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/051ec3165e95434eb7686c61f00828bc] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp, totalSize=36.9 K 2024-12-03T02:33:49,151 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 3303b10a8d5f441e8b5c7e9b0527f9a0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1733193225401 2024-12-03T02:33:49,152 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 630d75c574384dffa7a9c52ba7bfedad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1733193226627 2024-12-03T02:33:49,154 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 051ec3165e95434eb7686c61f00828bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=624, earliestPutTs=1733193227819 2024-12-03T02:33:49,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741924_1100 (size=12301) 2024-12-03T02:33:49,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/96059f17688c44bd9ef637a06b5c65db 2024-12-03T02:33:49,177 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2199fc0aeeef0203e54ad44ea3ac1a7#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:49,177 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/1be900834d894224b27e19aa6ddf5517 is 50, key is test_row_0/C:col10/1733193227819/Put/seqid=0 2024-12-03T02:33:49,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/651340cf513044fba22b08d32934251e is 50, key is test_row_0/C:col10/1733193228544/Put/seqid=0 2024-12-03T02:33:49,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741926_1102 (size=12301) 2024-12-03T02:33:49,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741925_1101 (size=13255) 2024-12-03T02:33:49,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=644 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/651340cf513044fba22b08d32934251e 2024-12-03T02:33:49,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/e0b4a4a7d4494d9b96e66a18a3bf6a3e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e0b4a4a7d4494d9b96e66a18a3bf6a3e 2024-12-03T02:33:49,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e0b4a4a7d4494d9b96e66a18a3bf6a3e, entries=200, sequenceid=644, filesize=14.4 K 2024-12-03T02:33:49,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/96059f17688c44bd9ef637a06b5c65db as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96059f17688c44bd9ef637a06b5c65db 2024-12-03T02:33:49,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96059f17688c44bd9ef637a06b5c65db, entries=150, sequenceid=644, filesize=12.0 K 2024-12-03T02:33:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/651340cf513044fba22b08d32934251e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/651340cf513044fba22b08d32934251e 2024-12-03T02:33:49,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/651340cf513044fba22b08d32934251e, entries=150, sequenceid=644, filesize=12.0 K 2024-12-03T02:33:49,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 551ms, sequenceid=644, compaction requested=false 2024-12-03T02:33:49,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T02:33:49,295 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:49,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T02:33:49,346 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-03T02:33:49,346 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:33:49,346 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-03T02:33:49,346 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,346 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:33:49,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:49,346 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,346 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:49,347 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:49,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:49,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:49,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:49,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/3b3fd40701ce4dea853d5b5e7eee9f63 is 50, key is test_row_0/A:col10/1733193228691/Put/seqid=0 2024-12-03T02:33:49,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741927_1103 (size=12301) 2024-12-03T02:33:49,603 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/1be900834d894224b27e19aa6ddf5517 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/1be900834d894224b27e19aa6ddf5517 2024-12-03T02:33:49,610 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2199fc0aeeef0203e54ad44ea3ac1a7/C of a2199fc0aeeef0203e54ad44ea3ac1a7 into 1be900834d894224b27e19aa6ddf5517(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:49,611 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:49,611 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., storeName=a2199fc0aeeef0203e54ad44ea3ac1a7/C, priority=13, startTime=1733193228611; duration=0sec 2024-12-03T02:33:49,611 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:49,611 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:C 2024-12-03T02:33:49,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/3b3fd40701ce4dea853d5b5e7eee9f63 2024-12-03T02:33:49,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/d0b990877ff24a01991b7897e345e61e is 50, key is test_row_0/B:col10/1733193228691/Put/seqid=0 2024-12-03T02:33:49,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741928_1104 (size=12301) 2024-12-03T02:33:49,977 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-03T02:33:49,977 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:33:49,977 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,977 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,988 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-03T02:33:49,988 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-03T02:33:49,988 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:33:49,988 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:33:49,988 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,988 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,988 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,988 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,989 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2017 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1909 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 955 2024-12-03T02:33:49,990 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2864 rows 2024-12-03T02:33:49,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 959 2024-12-03T02:33:49,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2872 rows 2024-12-03T02:33:49,991 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:33:49,991 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:33:49,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:49,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:33:49,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T02:33:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:50,003 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T02:33:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T02:33:50,004 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193230004"}]},"ts":"1733193230004"} 2024-12-03T02:33:50,007 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T02:33:50,007 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T02:33:50,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T02:33:50,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2199fc0aeeef0203e54ad44ea3ac1a7, UNASSIGN}] 2024-12-03T02:33:50,016 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2199fc0aeeef0203e54ad44ea3ac1a7, UNASSIGN 2024-12-03T02:33:50,017 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a2199fc0aeeef0203e54ad44ea3ac1a7, regionState=CLOSING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:50,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2199fc0aeeef0203e54ad44ea3ac1a7, UNASSIGN because future has completed 2024-12-03T02:33:50,020 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:33:50,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:33:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T02:33:50,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/d0b990877ff24a01991b7897e345e61e 2024-12-03T02:33:50,179 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:50,179 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:33:50,180 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing a2199fc0aeeef0203e54ad44ea3ac1a7, disabling compactions & flushes 2024-12-03T02:33:50,181 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:50,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/28451981d4074fa094802d640225d39a is 50, key is test_row_0/C:col10/1733193228691/Put/seqid=0 2024-12-03T02:33:50,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741929_1105 (size=12301) 2024-12-03T02:33:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T02:33:50,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/28451981d4074fa094802d640225d39a 2024-12-03T02:33:50,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/3b3fd40701ce4dea853d5b5e7eee9f63 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3b3fd40701ce4dea853d5b5e7eee9f63 2024-12-03T02:33:50,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3b3fd40701ce4dea853d5b5e7eee9f63, entries=150, sequenceid=661, filesize=12.0 K 2024-12-03T02:33:50,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/d0b990877ff24a01991b7897e345e61e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/d0b990877ff24a01991b7897e345e61e 2024-12-03T02:33:50,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/d0b990877ff24a01991b7897e345e61e, entries=150, sequenceid=661, filesize=12.0 K 2024-12-03T02:33:50,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/28451981d4074fa094802d640225d39a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/28451981d4074fa094802d640225d39a 2024-12-03T02:33:50,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/28451981d4074fa094802d640225d39a, entries=150, sequenceid=661, filesize=12.0 K 2024-12-03T02:33:50,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 1267ms, sequenceid=661, compaction requested=true 2024-12-03T02:33:50,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2199fc0aeeef0203e54ad44ea3ac1a7: 2024-12-03T02:33:50,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:50,613 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:50,614 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:50,614 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. after waiting 0 ms 2024-12-03T02:33:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:50,614 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2199fc0aeeef0203e54ad44ea3ac1a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:50,614 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. because compaction request was cancelled 2024-12-03T02:33:50,614 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. because compaction request was cancelled 2024-12-03T02:33:50,614 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:B 2024-12-03T02:33:50,614 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:A 2024-12-03T02:33:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:50,614 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. because compaction request was cancelled 2024-12-03T02:33:50,614 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2199fc0aeeef0203e54ad44ea3ac1a7:C 2024-12-03T02:33:50,614 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing a2199fc0aeeef0203e54ad44ea3ac1a7 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-03T02:33:50,614 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=A 2024-12-03T02:33:50,614 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:50,614 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=B 2024-12-03T02:33:50,614 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:50,614 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2199fc0aeeef0203e54ad44ea3ac1a7, store=C 2024-12-03T02:33:50,614 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:50,618 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/9b41217f79564b948b934394d21d75ef is 50, key is test_row_1/A:col10/1733193229975/Put/seqid=0 2024-12-03T02:33:50,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741930_1106 (size=9857) 2024-12-03T02:33:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T02:33:51,024 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/9b41217f79564b948b934394d21d75ef 2024-12-03T02:33:51,038 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/5db2732993ec452499956c4663c4d737 is 50, key is test_row_1/B:col10/1733193229975/Put/seqid=0 2024-12-03T02:33:51,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741931_1107 (size=9857) 2024-12-03T02:33:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T02:33:51,444 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/5db2732993ec452499956c4663c4d737 2024-12-03T02:33:51,456 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/46a1c674bd914f64bc280c6ea3896c63 is 50, key is test_row_1/C:col10/1733193229975/Put/seqid=0 2024-12-03T02:33:51,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741932_1108 (size=9857) 2024-12-03T02:33:51,862 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/46a1c674bd914f64bc280c6ea3896c63 2024-12-03T02:33:51,871 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/A/9b41217f79564b948b934394d21d75ef as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/9b41217f79564b948b934394d21d75ef 2024-12-03T02:33:51,878 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/9b41217f79564b948b934394d21d75ef, entries=100, sequenceid=668, filesize=9.6 K 2024-12-03T02:33:51,879 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/B/5db2732993ec452499956c4663c4d737 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/5db2732993ec452499956c4663c4d737 2024-12-03T02:33:51,884 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/5db2732993ec452499956c4663c4d737, entries=100, sequenceid=668, filesize=9.6 K 2024-12-03T02:33:51,885 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/.tmp/C/46a1c674bd914f64bc280c6ea3896c63 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/46a1c674bd914f64bc280c6ea3896c63 2024-12-03T02:33:51,890 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/46a1c674bd914f64bc280c6ea3896c63, entries=100, sequenceid=668, filesize=9.6 K 2024-12-03T02:33:51,891 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 1277ms, sequenceid=668, compaction requested=true 2024-12-03T02:33:51,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/53cc22542fb340089bd241ccae2104b0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/cda043a1adc54167a032f336b682c406, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54f14c14520742c78037eaae62735227, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/1a5025f86f68425a922fcf22c4d29e73, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8a5b27458b3046f7b53001f6d59bb405, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7917efbf10e048159ba07980c2462114, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/b6fc4e833bfc403d8400b31c6efb8006, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/d1abae5bd7794876802532e8284846f3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c61554c61562494cb2d19c3ff3d48558, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/712c378865974b06ba931da8e32a129f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54b1b109862f422d9a895babc0c35785, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/eee44da8ec9940d7b9eef404e86316fb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/95d1377ef56742bfbf7dd3ec7a10305e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/15812b392cc54ca38caa5b2436980c08, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/4f94ed55141648c9a1c6aec104d24eea, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/2ba78ee35aec477e9ebeb439a16219b7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e85d1ba9797c4d02bd97e0ba5f91155d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/a07afe4316844d2ca2cd94c2087ac402, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/5dd3f506eda94ea68194356a7cde90c6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/84a99241b1d04f67a9078de612b0ed60, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/64ed31e7584a4a19846ef644774a4e3e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c866325165c14955b59c8885fb6cf150, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3ff79598a6ed4e0daa5293627cc90ad0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/47aec078898946d7b6e8169c01332d12, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/14632105951743d89eb32edb8c400045, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8f87eedd1ae345dca48fb65f764660bf, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/569856f0cfb94e749a05cf58a309fb36, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7afe53fab0de442d81b42ddde57e6b7b] to archive 2024-12-03T02:33:51,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:33:51,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/53cc22542fb340089bd241ccae2104b0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/53cc22542fb340089bd241ccae2104b0 2024-12-03T02:33:51,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/cda043a1adc54167a032f336b682c406 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/cda043a1adc54167a032f336b682c406 2024-12-03T02:33:51,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54f14c14520742c78037eaae62735227 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54f14c14520742c78037eaae62735227 2024-12-03T02:33:51,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/1a5025f86f68425a922fcf22c4d29e73 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/1a5025f86f68425a922fcf22c4d29e73 2024-12-03T02:33:51,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8a5b27458b3046f7b53001f6d59bb405 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8a5b27458b3046f7b53001f6d59bb405 2024-12-03T02:33:51,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7917efbf10e048159ba07980c2462114 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7917efbf10e048159ba07980c2462114 2024-12-03T02:33:51,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/b6fc4e833bfc403d8400b31c6efb8006 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/b6fc4e833bfc403d8400b31c6efb8006 2024-12-03T02:33:51,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/d1abae5bd7794876802532e8284846f3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/d1abae5bd7794876802532e8284846f3 2024-12-03T02:33:51,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c61554c61562494cb2d19c3ff3d48558 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c61554c61562494cb2d19c3ff3d48558 2024-12-03T02:33:51,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/712c378865974b06ba931da8e32a129f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/712c378865974b06ba931da8e32a129f 2024-12-03T02:33:51,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54b1b109862f422d9a895babc0c35785 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/54b1b109862f422d9a895babc0c35785 2024-12-03T02:33:51,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/eee44da8ec9940d7b9eef404e86316fb to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/eee44da8ec9940d7b9eef404e86316fb 2024-12-03T02:33:51,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/95d1377ef56742bfbf7dd3ec7a10305e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/95d1377ef56742bfbf7dd3ec7a10305e 2024-12-03T02:33:51,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/15812b392cc54ca38caa5b2436980c08 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/15812b392cc54ca38caa5b2436980c08 2024-12-03T02:33:51,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/4f94ed55141648c9a1c6aec104d24eea to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/4f94ed55141648c9a1c6aec104d24eea 2024-12-03T02:33:51,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/2ba78ee35aec477e9ebeb439a16219b7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/2ba78ee35aec477e9ebeb439a16219b7 2024-12-03T02:33:51,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e85d1ba9797c4d02bd97e0ba5f91155d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e85d1ba9797c4d02bd97e0ba5f91155d 2024-12-03T02:33:51,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/a07afe4316844d2ca2cd94c2087ac402 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/a07afe4316844d2ca2cd94c2087ac402 2024-12-03T02:33:51,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/5dd3f506eda94ea68194356a7cde90c6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/5dd3f506eda94ea68194356a7cde90c6 2024-12-03T02:33:51,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/84a99241b1d04f67a9078de612b0ed60 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/84a99241b1d04f67a9078de612b0ed60 2024-12-03T02:33:51,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/64ed31e7584a4a19846ef644774a4e3e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/64ed31e7584a4a19846ef644774a4e3e 2024-12-03T02:33:51,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c866325165c14955b59c8885fb6cf150 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/c866325165c14955b59c8885fb6cf150 2024-12-03T02:33:51,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3ff79598a6ed4e0daa5293627cc90ad0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3ff79598a6ed4e0daa5293627cc90ad0 2024-12-03T02:33:51,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/47aec078898946d7b6e8169c01332d12 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/47aec078898946d7b6e8169c01332d12 2024-12-03T02:33:51,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/14632105951743d89eb32edb8c400045 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/14632105951743d89eb32edb8c400045 2024-12-03T02:33:51,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8f87eedd1ae345dca48fb65f764660bf to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/8f87eedd1ae345dca48fb65f764660bf 2024-12-03T02:33:51,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/569856f0cfb94e749a05cf58a309fb36 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/569856f0cfb94e749a05cf58a309fb36 2024-12-03T02:33:51,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7afe53fab0de442d81b42ddde57e6b7b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/7afe53fab0de442d81b42ddde57e6b7b 2024-12-03T02:33:51,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/971906c5209844c8903c23baab9c2d97, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/270b32f8c00b4753afaa26598309fe67, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/a312dcf912ad4891a98da1e3b89a13ef, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ac1454a1e20e4cd3b5e8c39ce49466a9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0537fc216601434e8b165db152f7bb57, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/42ae3beeb5af42d89cc4cac3227b7460, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ba34090c9d054077bf15dbee468e096c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/aa8048d794f54db09e00e44f5289e8c7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/43704ccbad7f4caf90f820d6d08331be, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/777473178e4e419d9727bc7f2a9bd6b1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e6ecfcdd75f340d7ba27206593fc00a6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/302be0da1c4e418da5c81d80c6aa7cbf, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ad103dd4a89141a8885a9ec4f7a8b71c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/394838207210469ba7e2434680b12b8c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/b385de93a41c409981c8cb6bea47b794, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/f6e11d8a2fcc417b96649c43cbd7a7da, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/25f8be2a822d4c2bad8bda1b8e9eba42, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/3b7406a8474040b6b05a33a82461e51e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0b0841602fc9462daf5c4d40d10bbf5c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/679e88f6e4404265a8bb1269d42f1422, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/262d808343ed45229be225d26b391524, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e53708cca0c340659c27bd74b99cecd7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/673d7ae2362d4cd4b4c36efa6c93a266, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/546776c771b3450585632507458ec107, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/802c76f6311b4163b92e66171ad662c5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96d1d52f66474753b695f5f04a83c934, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/79b9fb5c13894267b25152cebb81051a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/c85b6de0c69b42b19ab0c1a8c93dc0d9] to archive 2024-12-03T02:33:51,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:33:51,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/971906c5209844c8903c23baab9c2d97 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/971906c5209844c8903c23baab9c2d97 2024-12-03T02:33:51,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/270b32f8c00b4753afaa26598309fe67 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/270b32f8c00b4753afaa26598309fe67 2024-12-03T02:33:51,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/a312dcf912ad4891a98da1e3b89a13ef to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/a312dcf912ad4891a98da1e3b89a13ef 2024-12-03T02:33:51,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ac1454a1e20e4cd3b5e8c39ce49466a9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ac1454a1e20e4cd3b5e8c39ce49466a9 2024-12-03T02:33:51,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0537fc216601434e8b165db152f7bb57 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0537fc216601434e8b165db152f7bb57 2024-12-03T02:33:51,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/42ae3beeb5af42d89cc4cac3227b7460 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/42ae3beeb5af42d89cc4cac3227b7460 2024-12-03T02:33:51,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ba34090c9d054077bf15dbee468e096c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ba34090c9d054077bf15dbee468e096c 2024-12-03T02:33:51,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/aa8048d794f54db09e00e44f5289e8c7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/aa8048d794f54db09e00e44f5289e8c7 2024-12-03T02:33:51,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/43704ccbad7f4caf90f820d6d08331be to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/43704ccbad7f4caf90f820d6d08331be 2024-12-03T02:33:51,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/777473178e4e419d9727bc7f2a9bd6b1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/777473178e4e419d9727bc7f2a9bd6b1 2024-12-03T02:33:51,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e6ecfcdd75f340d7ba27206593fc00a6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e6ecfcdd75f340d7ba27206593fc00a6 2024-12-03T02:33:51,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/302be0da1c4e418da5c81d80c6aa7cbf to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/302be0da1c4e418da5c81d80c6aa7cbf 2024-12-03T02:33:51,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ad103dd4a89141a8885a9ec4f7a8b71c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/ad103dd4a89141a8885a9ec4f7a8b71c 2024-12-03T02:33:51,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/394838207210469ba7e2434680b12b8c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/394838207210469ba7e2434680b12b8c 2024-12-03T02:33:51,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/b385de93a41c409981c8cb6bea47b794 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/b385de93a41c409981c8cb6bea47b794 2024-12-03T02:33:51,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/f6e11d8a2fcc417b96649c43cbd7a7da to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/f6e11d8a2fcc417b96649c43cbd7a7da 2024-12-03T02:33:51,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/25f8be2a822d4c2bad8bda1b8e9eba42 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/25f8be2a822d4c2bad8bda1b8e9eba42 2024-12-03T02:33:51,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/3b7406a8474040b6b05a33a82461e51e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/3b7406a8474040b6b05a33a82461e51e 2024-12-03T02:33:51,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0b0841602fc9462daf5c4d40d10bbf5c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/0b0841602fc9462daf5c4d40d10bbf5c 2024-12-03T02:33:51,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/679e88f6e4404265a8bb1269d42f1422 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/679e88f6e4404265a8bb1269d42f1422 2024-12-03T02:33:51,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/262d808343ed45229be225d26b391524 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/262d808343ed45229be225d26b391524 2024-12-03T02:33:51,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e53708cca0c340659c27bd74b99cecd7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/e53708cca0c340659c27bd74b99cecd7 2024-12-03T02:33:51,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/673d7ae2362d4cd4b4c36efa6c93a266 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/673d7ae2362d4cd4b4c36efa6c93a266 2024-12-03T02:33:51,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/546776c771b3450585632507458ec107 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/546776c771b3450585632507458ec107 2024-12-03T02:33:51,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/802c76f6311b4163b92e66171ad662c5 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/802c76f6311b4163b92e66171ad662c5 2024-12-03T02:33:51,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96d1d52f66474753b695f5f04a83c934 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96d1d52f66474753b695f5f04a83c934 2024-12-03T02:33:51,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/79b9fb5c13894267b25152cebb81051a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/79b9fb5c13894267b25152cebb81051a 2024-12-03T02:33:51,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/c85b6de0c69b42b19ab0c1a8c93dc0d9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/c85b6de0c69b42b19ab0c1a8c93dc0d9 2024-12-03T02:33:51,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/535b70fcce7047b7aac7f8acb9d2b4db, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/861cd31d00f14bd5ac90098662470204, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/193ae69d79074b18ae33fc053a47cb89, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/27317faa5cd544a38be56f9910c7b198, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/eb908facbe8744dc9e0f3c4551e97d96, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/be57be4a845148d89420c336e3e54a6f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/db459e52eb834626857050c7abf094e6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/5df1f8d81cee4a9dbfed5dd6e7eee8af, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/29ec8a0bbd544feeb7f1cd3b60eaa633, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/605a5a1704d244b8905e641e7a55d981, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ef7e75e2e9b640cbb64beb8209d2a319, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/d86d302a5f4d486f91ceaf9de6f27eb3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/31b935d0169944e09a0236283fa0acc7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/6da36179f470417087f729bfa44b9ba4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e25f891796a54e3ba8cbaf9d4a61629c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e48681d42e6c44159c5e06d8e2e8dc56, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ec36fbfc565a41e5aa6971acc2db1628, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ecd7c706f16849ebb86ca99c513a9500, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/2fb4bc820adb43a49c5a4f8c7c6610f1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/800fd19d253f40c5b3683b022dae532d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/7259e890654f4201b7871eeac9495f02, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/cfba5ca36aa54c41bf7daca2aa891439, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/839e98ebcb7c41af9b1eee8d4703f71e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/0a412b8c92194dd3b781560f9c50cf34, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3303b10a8d5f441e8b5c7e9b0527f9a0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3e4ac9ef0e664cf0b71c1e366c6e9752, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/630d75c574384dffa7a9c52ba7bfedad, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/051ec3165e95434eb7686c61f00828bc] to archive 2024-12-03T02:33:51,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:33:51,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/535b70fcce7047b7aac7f8acb9d2b4db to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/535b70fcce7047b7aac7f8acb9d2b4db 2024-12-03T02:33:51,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/861cd31d00f14bd5ac90098662470204 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/861cd31d00f14bd5ac90098662470204 2024-12-03T02:33:51,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/193ae69d79074b18ae33fc053a47cb89 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/193ae69d79074b18ae33fc053a47cb89 2024-12-03T02:33:51,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/27317faa5cd544a38be56f9910c7b198 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/27317faa5cd544a38be56f9910c7b198 2024-12-03T02:33:51,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/eb908facbe8744dc9e0f3c4551e97d96 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/eb908facbe8744dc9e0f3c4551e97d96 2024-12-03T02:33:51,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/be57be4a845148d89420c336e3e54a6f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/be57be4a845148d89420c336e3e54a6f 2024-12-03T02:33:51,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/db459e52eb834626857050c7abf094e6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/db459e52eb834626857050c7abf094e6 2024-12-03T02:33:51,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/5df1f8d81cee4a9dbfed5dd6e7eee8af to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/5df1f8d81cee4a9dbfed5dd6e7eee8af 2024-12-03T02:33:51,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/29ec8a0bbd544feeb7f1cd3b60eaa633 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/29ec8a0bbd544feeb7f1cd3b60eaa633 2024-12-03T02:33:51,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/605a5a1704d244b8905e641e7a55d981 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/605a5a1704d244b8905e641e7a55d981 2024-12-03T02:33:52,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ef7e75e2e9b640cbb64beb8209d2a319 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ef7e75e2e9b640cbb64beb8209d2a319 2024-12-03T02:33:52,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/d86d302a5f4d486f91ceaf9de6f27eb3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/d86d302a5f4d486f91ceaf9de6f27eb3 2024-12-03T02:33:52,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/31b935d0169944e09a0236283fa0acc7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/31b935d0169944e09a0236283fa0acc7 2024-12-03T02:33:52,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/6da36179f470417087f729bfa44b9ba4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/6da36179f470417087f729bfa44b9ba4 2024-12-03T02:33:52,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e25f891796a54e3ba8cbaf9d4a61629c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e25f891796a54e3ba8cbaf9d4a61629c 2024-12-03T02:33:52,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e48681d42e6c44159c5e06d8e2e8dc56 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/e48681d42e6c44159c5e06d8e2e8dc56 2024-12-03T02:33:52,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ec36fbfc565a41e5aa6971acc2db1628 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ec36fbfc565a41e5aa6971acc2db1628 2024-12-03T02:33:52,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ecd7c706f16849ebb86ca99c513a9500 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/ecd7c706f16849ebb86ca99c513a9500 2024-12-03T02:33:52,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/2fb4bc820adb43a49c5a4f8c7c6610f1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/2fb4bc820adb43a49c5a4f8c7c6610f1 2024-12-03T02:33:52,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/800fd19d253f40c5b3683b022dae532d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/800fd19d253f40c5b3683b022dae532d 2024-12-03T02:33:52,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/7259e890654f4201b7871eeac9495f02 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/7259e890654f4201b7871eeac9495f02 2024-12-03T02:33:52,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/cfba5ca36aa54c41bf7daca2aa891439 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/cfba5ca36aa54c41bf7daca2aa891439 2024-12-03T02:33:52,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/839e98ebcb7c41af9b1eee8d4703f71e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/839e98ebcb7c41af9b1eee8d4703f71e 2024-12-03T02:33:52,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/0a412b8c92194dd3b781560f9c50cf34 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/0a412b8c92194dd3b781560f9c50cf34 2024-12-03T02:33:52,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3303b10a8d5f441e8b5c7e9b0527f9a0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3303b10a8d5f441e8b5c7e9b0527f9a0 2024-12-03T02:33:52,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3e4ac9ef0e664cf0b71c1e366c6e9752 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/3e4ac9ef0e664cf0b71c1e366c6e9752 2024-12-03T02:33:52,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/630d75c574384dffa7a9c52ba7bfedad to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/630d75c574384dffa7a9c52ba7bfedad 2024-12-03T02:33:52,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/051ec3165e95434eb7686c61f00828bc to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/051ec3165e95434eb7686c61f00828bc 2024-12-03T02:33:52,025 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/recovered.edits/671.seqid, newMaxSeqId=671, maxSeqId=1 2024-12-03T02:33:52,025 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. 2024-12-03T02:33:52,026 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for a2199fc0aeeef0203e54ad44ea3ac1a7: Waiting for close lock at 1733193230180Running coprocessor pre-close hooks at 1733193230180Disabling compacts and flushes for region at 1733193230180Disabling writes for close at 1733193230614 (+434 ms)Obtaining lock to block concurrent updates at 1733193230614Preparing flush snapshotting stores in a2199fc0aeeef0203e54ad44ea3ac1a7 at 1733193230614Finished memstore snapshotting TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733193230614Flushing stores of TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7. at 1733193230615 (+1 ms)Flushing a2199fc0aeeef0203e54ad44ea3ac1a7/A: creating writer at 1733193230615Flushing a2199fc0aeeef0203e54ad44ea3ac1a7/A: appending metadata at 1733193230618 (+3 ms)Flushing a2199fc0aeeef0203e54ad44ea3ac1a7/A: closing flushed file at 1733193230618Flushing a2199fc0aeeef0203e54ad44ea3ac1a7/B: creating writer at 1733193231032 (+414 ms)Flushing a2199fc0aeeef0203e54ad44ea3ac1a7/B: appending metadata at 1733193231037 (+5 ms)Flushing a2199fc0aeeef0203e54ad44ea3ac1a7/B: closing flushed file at 1733193231037Flushing a2199fc0aeeef0203e54ad44ea3ac1a7/C: creating writer at 1733193231452 (+415 ms)Flushing a2199fc0aeeef0203e54ad44ea3ac1a7/C: appending metadata at 1733193231456 (+4 ms)Flushing a2199fc0aeeef0203e54ad44ea3ac1a7/C: closing flushed file at 1733193231456Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1deca485: reopening flushed file at 1733193231870 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@292497d2: reopening flushed file at 1733193231878 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40934a23: reopening flushed file at 1733193231884 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a2199fc0aeeef0203e54ad44ea3ac1a7 in 1277ms, sequenceid=668, compaction requested=true at 1733193231891 (+7 ms)Writing region close event to WAL at 1733193232021 (+130 ms)Running coprocessor post-close hooks at 1733193232025 (+4 ms)Closed at 1733193232025 2024-12-03T02:33:52,028 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:52,029 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a2199fc0aeeef0203e54ad44ea3ac1a7, regionState=CLOSED 2024-12-03T02:33:52,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:33:52,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T02:33:52,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure a2199fc0aeeef0203e54ad44ea3ac1a7, server=64035cbb0b46,38721,1733193202350 in 2.0140 sec 2024-12-03T02:33:52,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T02:33:52,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2199fc0aeeef0203e54ad44ea3ac1a7, UNASSIGN in 2.0230 sec 2024-12-03T02:33:52,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-03T02:33:52,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0320 sec 2024-12-03T02:33:52,044 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193232043"}]},"ts":"1733193232043"} 2024-12-03T02:33:52,045 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T02:33:52,045 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T02:33:52,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0480 sec 2024-12-03T02:33:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T02:33:52,166 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:52,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T02:33:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:52,178 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T02:33:52,181 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:52,187 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:52,191 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/recovered.edits] 2024-12-03T02:33:52,195 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3b3fd40701ce4dea853d5b5e7eee9f63 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/3b3fd40701ce4dea853d5b5e7eee9f63 2024-12-03T02:33:52,197 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/591675012b98496f8c0026085167bbf3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/591675012b98496f8c0026085167bbf3 2024-12-03T02:33:52,198 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/9b41217f79564b948b934394d21d75ef to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/9b41217f79564b948b934394d21d75ef 2024-12-03T02:33:52,200 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e0b4a4a7d4494d9b96e66a18a3bf6a3e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/A/e0b4a4a7d4494d9b96e66a18a3bf6a3e 2024-12-03T02:33:52,204 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/19cfe2f35f1c43a1ac3319ab2e6ed365 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/19cfe2f35f1c43a1ac3319ab2e6ed365 2024-12-03T02:33:52,206 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/5db2732993ec452499956c4663c4d737 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/5db2732993ec452499956c4663c4d737 2024-12-03T02:33:52,208 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96059f17688c44bd9ef637a06b5c65db to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/96059f17688c44bd9ef637a06b5c65db 2024-12-03T02:33:52,209 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/d0b990877ff24a01991b7897e345e61e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/B/d0b990877ff24a01991b7897e345e61e 2024-12-03T02:33:52,212 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/1be900834d894224b27e19aa6ddf5517 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/1be900834d894224b27e19aa6ddf5517 2024-12-03T02:33:52,214 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/28451981d4074fa094802d640225d39a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/28451981d4074fa094802d640225d39a 2024-12-03T02:33:52,217 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/46a1c674bd914f64bc280c6ea3896c63 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/46a1c674bd914f64bc280c6ea3896c63 2024-12-03T02:33:52,219 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/651340cf513044fba22b08d32934251e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/C/651340cf513044fba22b08d32934251e 2024-12-03T02:33:52,224 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/recovered.edits/671.seqid to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7/recovered.edits/671.seqid 2024-12-03T02:33:52,226 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/a2199fc0aeeef0203e54ad44ea3ac1a7 2024-12-03T02:33:52,226 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T02:33:52,232 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:52,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-03T02:33:52,245 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T02:33:52,274 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T02:33:52,276 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:52,276 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T02:33:52,277 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733193232276"}]},"ts":"9223372036854775807"} 2024-12-03T02:33:52,281 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T02:33:52,281 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a2199fc0aeeef0203e54ad44ea3ac1a7, NAME => 'TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T02:33:52,281 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T02:33:52,281 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733193232281"}]},"ts":"9223372036854775807"} 2024-12-03T02:33:52,284 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T02:33:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T02:33:52,285 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:52,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 112 msec 2024-12-03T02:33:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T02:33:52,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T02:33:52,496 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:52,509 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_756972765_22 at /127.0.0.1:44306 [Waiting for operation #263] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2037865029_22 at /127.0.0.1:51974 [Waiting for operation #80] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2037865029_22 at /127.0.0.1:46260 [Waiting for operation #252] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;64035cbb0b46:38721-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=913 (was 758) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2077 (was 3273) 2024-12-03T02:33:52,519 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=913, ProcessCount=11, AvailableMemoryMB=2076 2024-12-03T02:33:52,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T02:33:52,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:33:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:52,525 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T02:33:52,525 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:52,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-12-03T02:33:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T02:33:52,527 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T02:33:52,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741933_1109 (size=1061) 2024-12-03T02:33:52,544 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37 2024-12-03T02:33:52,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741934_1110 (size=53) 2024-12-03T02:33:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T02:33:52,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T02:33:52,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:33:52,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 64446d8651566fb34f3be9578452136e, disabling compactions & flushes 2024-12-03T02:33:52,969 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:52,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:52,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. after waiting 0 ms 2024-12-03T02:33:52,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:52,969 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:52,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 64446d8651566fb34f3be9578452136e: Waiting for close lock at 1733193232969Disabling compacts and flushes for region at 1733193232969Disabling writes for close at 1733193232969Writing region close event to WAL at 1733193232969Closed at 1733193232969 2024-12-03T02:33:52,971 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T02:33:52,971 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733193232971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733193232971"}]},"ts":"1733193232971"} 2024-12-03T02:33:52,974 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T02:33:52,976 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T02:33:52,976 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193232976"}]},"ts":"1733193232976"} 2024-12-03T02:33:52,978 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T02:33:52,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, ASSIGN}] 2024-12-03T02:33:52,979 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, ASSIGN 2024-12-03T02:33:52,980 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, ASSIGN; state=OFFLINE, location=64035cbb0b46,38721,1733193202350; forceNewPlan=false, retain=false 2024-12-03T02:33:53,131 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=64446d8651566fb34f3be9578452136e, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:53,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, ASSIGN because future has completed 2024-12-03T02:33:53,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:33:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T02:33:53,294 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:53,294 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:33:53,295 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,295 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:33:53,295 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,295 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,298 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,299 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:33:53,301 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64446d8651566fb34f3be9578452136e columnFamilyName A 2024-12-03T02:33:53,301 DEBUG [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:53,302 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(327): Store=64446d8651566fb34f3be9578452136e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:53,302 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,303 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:33:53,304 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64446d8651566fb34f3be9578452136e columnFamilyName B 2024-12-03T02:33:53,304 DEBUG [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:53,304 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(327): Store=64446d8651566fb34f3be9578452136e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:53,304 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,306 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:33:53,306 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64446d8651566fb34f3be9578452136e columnFamilyName C 2024-12-03T02:33:53,306 DEBUG [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:53,306 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(327): Store=64446d8651566fb34f3be9578452136e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:53,306 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:53,306 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,307 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,308 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,309 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,309 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,309 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:33:53,311 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,313 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:33:53,314 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 64446d8651566fb34f3be9578452136e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67246772, jitterRate=0.0020549893379211426}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:33:53,314 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:53,315 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 64446d8651566fb34f3be9578452136e: Running coprocessor pre-open hook at 1733193233296Writing region info on filesystem at 1733193233296Initializing all the Stores at 1733193233297 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193233297Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193233297Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193233297Cleaning up temporary data from old regions at 1733193233309 (+12 ms)Running coprocessor post-open hooks at 1733193233314 (+5 ms)Region opened successfully at 1733193233315 (+1 ms) 2024-12-03T02:33:53,316 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., pid=28, masterSystemTime=1733193233290 2024-12-03T02:33:53,319 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:53,319 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:53,320 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=64446d8651566fb34f3be9578452136e, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:53,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:33:53,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-03T02:33:53,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 in 188 msec 2024-12-03T02:33:53,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T02:33:53,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, ASSIGN in 349 msec 2024-12-03T02:33:53,329 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T02:33:53,330 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193233329"}]},"ts":"1733193233329"} 2024-12-03T02:33:53,331 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T02:33:53,333 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T02:33:53,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 811 msec 2024-12-03T02:33:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T02:33:53,666 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:53,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:53,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:53,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:53,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c300ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:53,670 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:53,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230c8e56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:53,672 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:53,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:53,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:53,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:53,676 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T02:33:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 64035cbb0b46,44739,1733193201066 2024-12-03T02:33:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d9da2d8 2024-12-03T02:33:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T02:33:53,679 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T02:33:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T02:33:53,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:33:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T02:33:53,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741935_1111 (size=1097) 2024-12-03T02:33:54,107 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T02:33:54,107 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T02:33:54,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T02:33:54,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, REOPEN/MOVE}] 2024-12-03T02:33:54,122 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, REOPEN/MOVE 2024-12-03T02:33:54,123 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=64446d8651566fb34f3be9578452136e, regionState=CLOSING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:54,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, REOPEN/MOVE because future has completed 2024-12-03T02:33:54,125 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:33:54,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:33:54,280 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,280 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:33:54,280 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 64446d8651566fb34f3be9578452136e, disabling compactions & flushes 2024-12-03T02:33:54,280 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,280 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,280 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. after waiting 0 ms 2024-12-03T02:33:54,280 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,287 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T02:33:54,288 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,288 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 64446d8651566fb34f3be9578452136e: Waiting for close lock at 1733193234280Running coprocessor pre-close hooks at 1733193234280Disabling compacts and flushes for region at 1733193234280Disabling writes for close at 1733193234280Writing region close event to WAL at 1733193234281 (+1 ms)Running coprocessor post-close hooks at 1733193234288 (+7 ms)Closed at 1733193234288 2024-12-03T02:33:54,288 WARN [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 64446d8651566fb34f3be9578452136e to self. 2024-12-03T02:33:54,292 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,293 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=64446d8651566fb34f3be9578452136e, regionState=CLOSED 2024-12-03T02:33:54,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:33:54,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T02:33:54,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 in 170 msec 2024-12-03T02:33:54,299 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, REOPEN/MOVE; state=CLOSED, location=64035cbb0b46,38721,1733193202350; forceNewPlan=false, retain=true 2024-12-03T02:33:54,450 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=64446d8651566fb34f3be9578452136e, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:54,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, REOPEN/MOVE because future has completed 2024-12-03T02:33:54,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:33:54,610 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,611 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:33:54,611 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,611 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:33:54,611 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,611 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,616 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,617 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:33:54,622 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64446d8651566fb34f3be9578452136e columnFamilyName A 2024-12-03T02:33:54,625 DEBUG [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:54,625 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(327): Store=64446d8651566fb34f3be9578452136e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:54,626 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,627 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:33:54,627 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64446d8651566fb34f3be9578452136e columnFamilyName B 2024-12-03T02:33:54,627 DEBUG [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:54,627 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(327): Store=64446d8651566fb34f3be9578452136e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:54,628 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,628 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:33:54,628 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64446d8651566fb34f3be9578452136e columnFamilyName C 2024-12-03T02:33:54,628 DEBUG [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:54,629 INFO [StoreOpener-64446d8651566fb34f3be9578452136e-1 {}] regionserver.HStore(327): Store=64446d8651566fb34f3be9578452136e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:33:54,629 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,629 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,630 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,631 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,632 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,632 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,632 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:33:54,634 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,635 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 64446d8651566fb34f3be9578452136e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60801564, jitterRate=-0.09398609399795532}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:33:54,635 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,636 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 64446d8651566fb34f3be9578452136e: Running coprocessor pre-open hook at 1733193234611Writing region info on filesystem at 1733193234611Initializing all the Stores at 1733193234612 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193234612Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193234613 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193234613Cleaning up temporary data from old regions at 1733193234632 (+19 ms)Running coprocessor post-open hooks at 1733193234635 (+3 ms)Region opened successfully at 1733193234636 (+1 ms) 2024-12-03T02:33:54,636 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., pid=33, masterSystemTime=1733193234607 2024-12-03T02:33:54,639 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,639 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,640 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=64446d8651566fb34f3be9578452136e, regionState=OPEN, openSeqNum=5, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:54,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:33:54,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-12-03T02:33:54,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 in 192 msec 2024-12-03T02:33:54,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-03T02:33:54,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, REOPEN/MOVE in 529 msec 2024-12-03T02:33:54,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T02:33:54,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 544 msec 2024-12-03T02:33:54,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 972 msec 2024-12-03T02:33:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T02:33:54,665 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:54,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:54,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:54,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@507b93a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:54,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:54,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:54,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5298f036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:54,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:54,677 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a8b0230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:54,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:54,683 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287d5f94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:54,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:54,689 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:54,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:54,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36843e65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:54,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:54,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:54,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@527427f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:54,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:54,698 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121017a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:54,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:54,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:54,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522d6c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:54,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:54,703 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201bad8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:54,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:54,705 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f674cc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:54,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:54,708 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aacf2b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:54,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:54,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:54,711 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b8d7648, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:54,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:54,721 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335d07fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:54,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e4b293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:54,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:54,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:54,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@691e4cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:54,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:54,728 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292c9d41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,729 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:33:54,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:33:54,730 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:33:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:33:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:33:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f49522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:33:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:33:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:33:54,732 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:33:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6668979c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:33:54,734 DEBUG [Thread-584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:54,734 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:54,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:33:54,734 DEBUG [Thread-577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:54,735 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:54,735 DEBUG [Thread-575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-12-03T02:33:54,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:54,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:54,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:54,737 DEBUG [Thread-581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:54,738 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:54,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T02:33:54,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:54,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:54,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:54,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:54,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:33:54,740 DEBUG [Thread-579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:54,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:54,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:54,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:54,742 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:33:54,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:33:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:33:54,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:33:54,745 DEBUG [Thread-573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:33:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:33:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:33:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:33:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:33:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:33:54,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:33:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:33:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:33:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:33:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:33:54,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:33:54,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:33:54,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:54,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:33:54,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:54,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:33:54,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f084ca92e6ec4d15b867794e88492eef_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193234762/Put/seqid=0 2024-12-03T02:33:54,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733193234788, max timestamp=1733193234821Num uniques 859; 2024-12-03T02:33:54,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733193234788, max timestamp=1733193234821Num uniques 859; 2024-12-03T02:33:54,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-03T02:33:54,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-03T02:33:54,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733193234788, max timestamp=1733193234821Num uniques 859; 2024-12-03T02:33:54,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-03T02:33:54,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741936_1112 (size=19474) 2024-12-03T02:33:54,841 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T02:33:54,848 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f084ca92e6ec4d15b867794e88492eef_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f084ca92e6ec4d15b867794e88492eef_64446d8651566fb34f3be9578452136e 2024-12-03T02:33:54,850 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/eb91356c5e6e4490afd493dc4201feb2, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:33:54,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/eb91356c5e6e4490afd493dc4201feb2 is 175, key is test_row_0/A:col10/1733193234762/Put/seqid=0 2024-12-03T02:33:54,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741937_1113 (size=56733) 2024-12-03T02:33:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:54,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193294843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193294849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:54,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193294882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:54,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193294883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:54,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193294895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:54,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:54,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:54,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193294985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193294995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193294995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193295005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193295018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T02:33:55,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193295195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193295214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193295214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:55,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:55,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193295225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193295225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,271 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/eb91356c5e6e4490afd493dc4201feb2 2024-12-03T02:33:55,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/c95c1bea69f04167b7bad7ff4dec07f9 is 50, key is test_row_0/B:col10/1733193234762/Put/seqid=0 2024-12-03T02:33:55,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741938_1114 (size=12001) 2024-12-03T02:33:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T02:33:55,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:55,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:55,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193295504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193295525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193295525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193295535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193295544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:55,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:55,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:55,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:55,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/c95c1bea69f04167b7bad7ff4dec07f9 2024-12-03T02:33:55,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/1d02acc8c10f45eab1a04e40a61cf71e is 50, key is test_row_0/C:col10/1733193234762/Put/seqid=0 2024-12-03T02:33:55,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741939_1115 (size=12001) 2024-12-03T02:33:55,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:55,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T02:33:56,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:56,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:56,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:56,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:56,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:56,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:56,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193296015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:56,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193296035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:56,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193296035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:56,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193296045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:56,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193296073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:56,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:56,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:56,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:56,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:56,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/1d02acc8c10f45eab1a04e40a61cf71e 2024-12-03T02:33:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/eb91356c5e6e4490afd493dc4201feb2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb91356c5e6e4490afd493dc4201feb2 2024-12-03T02:33:56,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb91356c5e6e4490afd493dc4201feb2, entries=300, sequenceid=14, filesize=55.4 K 2024-12-03T02:33:56,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/c95c1bea69f04167b7bad7ff4dec07f9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/c95c1bea69f04167b7bad7ff4dec07f9 2024-12-03T02:33:56,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/c95c1bea69f04167b7bad7ff4dec07f9, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T02:33:56,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/1d02acc8c10f45eab1a04e40a61cf71e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d02acc8c10f45eab1a04e40a61cf71e 2024-12-03T02:33:56,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d02acc8c10f45eab1a04e40a61cf71e, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T02:33:56,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 64446d8651566fb34f3be9578452136e in 1506ms, sequenceid=14, compaction requested=false 2024-12-03T02:33:56,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:33:56,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T02:33:56,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:56,316 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=147.60 KB heapSize=392.02 KB 2024-12-03T02:33:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:33:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:33:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:33:56,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:56,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032676ab70eef54d4c87d2b14a2a864374_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193234865/Put/seqid=0 2024-12-03T02:33:56,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741940_1116 (size=12154) 2024-12-03T02:33:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:56,764 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032676ab70eef54d4c87d2b14a2a864374_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032676ab70eef54d4c87d2b14a2a864374_64446d8651566fb34f3be9578452136e 2024-12-03T02:33:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/191e34ed0c1b4e15897f9ced4740add1, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:33:56,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/191e34ed0c1b4e15897f9ced4740add1 is 175, key is test_row_0/A:col10/1733193234865/Put/seqid=0 2024-12-03T02:33:56,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741941_1117 (size=30955) 2024-12-03T02:33:56,786 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=46, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/191e34ed0c1b4e15897f9ced4740add1 2024-12-03T02:33:56,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/7832afab0871420d99a59a049dd45773 is 50, key is test_row_0/B:col10/1733193234865/Put/seqid=0 2024-12-03T02:33:56,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741942_1118 (size=12001) 2024-12-03T02:33:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T02:33:57,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:57,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193297055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,059 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:33:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193297057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193297059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193297065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,066 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:33:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193297085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,091 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:33:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193297175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193297175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,207 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/7832afab0871420d99a59a049dd45773 2024-12-03T02:33:57,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/41a75a4b03744c749a6996de6fe07f79 is 50, key is test_row_0/C:col10/1733193234865/Put/seqid=0 2024-12-03T02:33:57,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741943_1119 (size=12001) 2024-12-03T02:33:57,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193297384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193297384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,574 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:33:57,622 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/41a75a4b03744c749a6996de6fe07f79 2024-12-03T02:33:57,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/191e34ed0c1b4e15897f9ced4740add1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/191e34ed0c1b4e15897f9ced4740add1 2024-12-03T02:33:57,637 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/191e34ed0c1b4e15897f9ced4740add1, entries=150, sequenceid=46, filesize=30.2 K 2024-12-03T02:33:57,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/7832afab0871420d99a59a049dd45773 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/7832afab0871420d99a59a049dd45773 2024-12-03T02:33:57,651 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/7832afab0871420d99a59a049dd45773, entries=150, sequenceid=46, filesize=11.7 K 2024-12-03T02:33:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/41a75a4b03744c749a6996de6fe07f79 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/41a75a4b03744c749a6996de6fe07f79 2024-12-03T02:33:57,663 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/41a75a4b03744c749a6996de6fe07f79, entries=150, sequenceid=46, filesize=11.7 K 2024-12-03T02:33:57,664 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~391.97 KB/401376, currentSize=40.25 KB/41220 for 64446d8651566fb34f3be9578452136e in 1348ms, sequenceid=46, compaction requested=false 2024-12-03T02:33:57,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:33:57,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:57,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-03T02:33:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-03T02:33:57,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-03T02:33:57,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9240 sec 2024-12-03T02:33:57,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.9370 sec 2024-12-03T02:33:57,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:33:57,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:33:57,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:57,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:33:57,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:57,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:33:57,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:57,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032ca2f564fca344b090beceecc66ae12a_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193237055/Put/seqid=0 2024-12-03T02:33:57,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741944_1120 (size=17034) 2024-12-03T02:33:57,734 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:57,741 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032ca2f564fca344b090beceecc66ae12a_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032ca2f564fca344b090beceecc66ae12a_64446d8651566fb34f3be9578452136e 2024-12-03T02:33:57,742 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/920b21e3e1264cf1bf2f96626b55ac8a, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:33:57,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/920b21e3e1264cf1bf2f96626b55ac8a is 175, key is test_row_0/A:col10/1733193237055/Put/seqid=0 2024-12-03T02:33:57,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193237724, max timestamp=1733193237748Num uniques 859; 2024-12-03T02:33:57,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:57,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193237724, max timestamp=1733193237748Num uniques 859; 2024-12-03T02:33:57,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193237724, max timestamp=1733193237748Num uniques 859; 2024-12-03T02:33:57,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:57,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:33:57,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741945_1121 (size=48139) 2024-12-03T02:33:57,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193297789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193297790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193297894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193297905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:57,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:58,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193298104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:58,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193298115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:58,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:58,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:58,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:58,164 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=57, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/920b21e3e1264cf1bf2f96626b55ac8a 2024-12-03T02:33:58,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/e869cdcb4d2149e1b04742f4cd343c2b is 50, key is test_row_0/B:col10/1733193237055/Put/seqid=0 2024-12-03T02:33:58,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741946_1122 (size=12001) 2024-12-03T02:33:58,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193298424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:58,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193298424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:58,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/e869cdcb4d2149e1b04742f4cd343c2b 2024-12-03T02:33:58,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/d67ff727d118430c958b7ba0dffa9d96 is 50, key is test_row_0/C:col10/1733193237055/Put/seqid=0 2024-12-03T02:33:58,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741947_1123 (size=12001) 2024-12-03T02:33:58,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/d67ff727d118430c958b7ba0dffa9d96 2024-12-03T02:33:58,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/920b21e3e1264cf1bf2f96626b55ac8a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/920b21e3e1264cf1bf2f96626b55ac8a 2024-12-03T02:33:58,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/920b21e3e1264cf1bf2f96626b55ac8a, entries=250, sequenceid=57, filesize=47.0 K 2024-12-03T02:33:58,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/e869cdcb4d2149e1b04742f4cd343c2b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e869cdcb4d2149e1b04742f4cd343c2b 2024-12-03T02:33:58,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e869cdcb4d2149e1b04742f4cd343c2b, entries=150, sequenceid=57, filesize=11.7 K 2024-12-03T02:33:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/d67ff727d118430c958b7ba0dffa9d96 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/d67ff727d118430c958b7ba0dffa9d96 2024-12-03T02:33:58,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/d67ff727d118430c958b7ba0dffa9d96, entries=150, sequenceid=57, filesize=11.7 K 2024-12-03T02:33:58,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 64446d8651566fb34f3be9578452136e in 967ms, sequenceid=57, compaction requested=true 2024-12-03T02:33:58,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:33:58,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:33:58,665 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:58,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:58,665 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:58,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:33:58,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:58,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:33:58,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:58,680 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:58,680 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/B is initiating minor compaction (all files) 2024-12-03T02:33:58,680 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/B in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:58,680 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/c95c1bea69f04167b7bad7ff4dec07f9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/7832afab0871420d99a59a049dd45773, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e869cdcb4d2149e1b04742f4cd343c2b] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=35.2 K 2024-12-03T02:33:58,681 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:58,681 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/A is initiating minor compaction (all files) 2024-12-03T02:33:58,681 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/A in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:58,681 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting c95c1bea69f04167b7bad7ff4dec07f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733193234762 2024-12-03T02:33:58,681 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb91356c5e6e4490afd493dc4201feb2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/191e34ed0c1b4e15897f9ced4740add1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/920b21e3e1264cf1bf2f96626b55ac8a] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=132.6 K 2024-12-03T02:33:58,681 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:58,681 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb91356c5e6e4490afd493dc4201feb2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/191e34ed0c1b4e15897f9ced4740add1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/920b21e3e1264cf1bf2f96626b55ac8a] 2024-12-03T02:33:58,682 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb91356c5e6e4490afd493dc4201feb2, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733193234757 2024-12-03T02:33:58,682 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7832afab0871420d99a59a049dd45773, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733193234847 2024-12-03T02:33:58,682 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e869cdcb4d2149e1b04742f4cd343c2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733193237055 2024-12-03T02:33:58,682 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 191e34ed0c1b4e15897f9ced4740add1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733193234847 2024-12-03T02:33:58,683 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 920b21e3e1264cf1bf2f96626b55ac8a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733193237050 2024-12-03T02:33:58,717 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:33:58,725 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:58,726 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/52465c021d1946aa8478911ace9097cb is 50, key is test_row_0/B:col10/1733193237055/Put/seqid=0 2024-12-03T02:33:58,736 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203ad1cc74945a340d980e95023198414c8_64446d8651566fb34f3be9578452136e store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:33:58,741 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203ad1cc74945a340d980e95023198414c8_64446d8651566fb34f3be9578452136e, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:33:58,741 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ad1cc74945a340d980e95023198414c8_64446d8651566fb34f3be9578452136e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:33:58,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741948_1124 (size=12104) 2024-12-03T02:33:58,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741949_1125 (size=4469) 2024-12-03T02:33:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T02:33:58,908 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:33:58,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:33:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-03T02:33:58,912 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:33:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T02:33:58,914 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:33:58,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:33:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:33:58,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-03T02:33:58,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:33:58,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:58,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:33:58,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:58,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:33:58,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:33:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T02:33:59,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193299017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193299026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120341c22be1521c4c5f91a32c4a74d499fa_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193238956/Put/seqid=0 2024-12-03T02:33:59,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T02:33:59,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193299065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,068 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:33:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741950_1126 (size=14594) 2024-12-03T02:33:59,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193299085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,092 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:33:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193299105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,111 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:33:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193299136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193299136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,196 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#A#compaction#105 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:59,197 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/7d7d283607894ea2bca81790ce4d0283 is 175, key is test_row_0/A:col10/1733193237055/Put/seqid=0 2024-12-03T02:33:59,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741951_1127 (size=31058) 2024-12-03T02:33:59,214 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/52465c021d1946aa8478911ace9097cb as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/52465c021d1946aa8478911ace9097cb 2024-12-03T02:33:59,224 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/7d7d283607894ea2bca81790ce4d0283 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7d7d283607894ea2bca81790ce4d0283 2024-12-03T02:33:59,225 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/B of 64446d8651566fb34f3be9578452136e into 52465c021d1946aa8478911ace9097cb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:59,225 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:33:59,225 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/B, priority=13, startTime=1733193238665; duration=0sec 2024-12-03T02:33:59,226 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:33:59,226 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:B 2024-12-03T02:33:59,226 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:33:59,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T02:33:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,233 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:33:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T02:33:59,236 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/C is initiating minor compaction (all files) 2024-12-03T02:33:59,236 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/C in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,236 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d02acc8c10f45eab1a04e40a61cf71e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/41a75a4b03744c749a6996de6fe07f79, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/d67ff727d118430c958b7ba0dffa9d96] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=35.2 K 2024-12-03T02:33:59,237 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/A of 64446d8651566fb34f3be9578452136e into 7d7d283607894ea2bca81790ce4d0283(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:59,237 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d02acc8c10f45eab1a04e40a61cf71e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733193234762 2024-12-03T02:33:59,237 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:33:59,237 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/A, priority=13, startTime=1733193238665; duration=0sec 2024-12-03T02:33:59,237 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:59,237 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:A 2024-12-03T02:33:59,237 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a75a4b03744c749a6996de6fe07f79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733193234847 2024-12-03T02:33:59,238 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d67ff727d118430c958b7ba0dffa9d96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733193237055 2024-12-03T02:33:59,281 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#C#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:33:59,281 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/58c6024ad94147a2bb4ec4a96fbe3b3c is 50, key is test_row_0/C:col10/1733193237055/Put/seqid=0 2024-12-03T02:33:59,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193299346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193299346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741952_1128 (size=12104) 2024-12-03T02:33:59,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T02:33:59,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:59,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,402 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/58c6024ad94147a2bb4ec4a96fbe3b3c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/58c6024ad94147a2bb4ec4a96fbe3b3c 2024-12-03T02:33:59,422 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/C of 64446d8651566fb34f3be9578452136e into 58c6024ad94147a2bb4ec4a96fbe3b3c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:33:59,422 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:33:59,422 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/C, priority=13, startTime=1733193238665; duration=0sec 2024-12-03T02:33:59,422 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:33:59,422 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:C 2024-12-03T02:33:59,488 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:59,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120341c22be1521c4c5f91a32c4a74d499fa_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120341c22be1521c4c5f91a32c4a74d499fa_64446d8651566fb34f3be9578452136e 2024-12-03T02:33:59,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/fbe4a3024087488bae9ad17a656be48f, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:33:59,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/fbe4a3024087488bae9ad17a656be48f is 175, key is test_row_0/A:col10/1733193238956/Put/seqid=0 2024-12-03T02:33:59,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T02:33:59,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:33:59,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T02:33:59,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:33:59,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741953_1129 (size=39549) 2024-12-03T02:33:59,548 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/fbe4a3024087488bae9ad17a656be48f 2024-12-03T02:33:59,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/e935f79f20ce4f87a1ab42de3b17697d is 50, key is test_row_0/B:col10/1733193238956/Put/seqid=0 2024-12-03T02:33:59,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741954_1130 (size=12001) 2024-12-03T02:33:59,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/e935f79f20ce4f87a1ab42de3b17697d 2024-12-03T02:33:59,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/19a0253660924708a37fa8e32055d2c1 is 50, key is test_row_0/C:col10/1733193238956/Put/seqid=0 2024-12-03T02:33:59,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741955_1131 (size=12001) 2024-12-03T02:33:59,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/19a0253660924708a37fa8e32055d2c1 2024-12-03T02:33:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/fbe4a3024087488bae9ad17a656be48f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/fbe4a3024087488bae9ad17a656be48f 2024-12-03T02:33:59,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/fbe4a3024087488bae9ad17a656be48f, entries=200, sequenceid=94, filesize=38.6 K 2024-12-03T02:33:59,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193299655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/e935f79f20ce4f87a1ab42de3b17697d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e935f79f20ce4f87a1ab42de3b17697d 2024-12-03T02:33:59,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:33:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193299665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:33:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:33:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:33:59,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e935f79f20ce4f87a1ab42de3b17697d, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T02:33:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/19a0253660924708a37fa8e32055d2c1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/19a0253660924708a37fa8e32055d2c1 2024-12-03T02:33:59,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/19a0253660924708a37fa8e32055d2c1, entries=150, sequenceid=94, filesize=11.7 K 2024-12-03T02:33:59,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 64446d8651566fb34f3be9578452136e in 728ms, sequenceid=94, compaction requested=false 2024-12-03T02:33:59,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:33:59,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T02:33:59,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:33:59,699 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T02:33:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:33:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:33:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:33:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:33:59,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030af4f7a64cd8432681af55f419a8f251_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193239024/Put/seqid=0 2024-12-03T02:33:59,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741956_1132 (size=12154) 2024-12-03T02:33:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:33:59,746 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030af4f7a64cd8432681af55f419a8f251_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030af4f7a64cd8432681af55f419a8f251_64446d8651566fb34f3be9578452136e 2024-12-03T02:33:59,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/eb6bc91600204400b5221afe46976ad6, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:33:59,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/eb6bc91600204400b5221afe46976ad6 is 175, key is test_row_0/A:col10/1733193239024/Put/seqid=0 2024-12-03T02:33:59,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741957_1133 (size=30955) 2024-12-03T02:33:59,774 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/eb6bc91600204400b5221afe46976ad6 2024-12-03T02:33:59,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/5b2652c21d3f4607a6b48eb14b920006 is 50, key is test_row_0/B:col10/1733193239024/Put/seqid=0 2024-12-03T02:33:59,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741958_1134 (size=12001) 2024-12-03T02:34:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T02:34:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:00,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:00,199 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/5b2652c21d3f4607a6b48eb14b920006 2024-12-03T02:34:00,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/7ac40aeb8b8b45778453791b64bd3670 is 50, key is test_row_0/C:col10/1733193239024/Put/seqid=0 2024-12-03T02:34:00,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193240200, max timestamp=1733193240219Num uniques 859; 2024-12-03T02:34:00,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:00,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193240200, max timestamp=1733193240219Num uniques 859; 2024-12-03T02:34:00,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193240200, max timestamp=1733193240219Num uniques 859; 2024-12-03T02:34:00,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:00,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:00,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741959_1135 (size=12001) 2024-12-03T02:34:00,230 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/7ac40aeb8b8b45778453791b64bd3670 2024-12-03T02:34:00,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/eb6bc91600204400b5221afe46976ad6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb6bc91600204400b5221afe46976ad6 2024-12-03T02:34:00,249 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb6bc91600204400b5221afe46976ad6, entries=150, sequenceid=105, filesize=30.2 K 2024-12-03T02:34:00,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/5b2652c21d3f4607a6b48eb14b920006 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5b2652c21d3f4607a6b48eb14b920006 2024-12-03T02:34:00,261 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5b2652c21d3f4607a6b48eb14b920006, entries=150, sequenceid=105, filesize=11.7 K 2024-12-03T02:34:00,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/7ac40aeb8b8b45778453791b64bd3670 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/7ac40aeb8b8b45778453791b64bd3670 2024-12-03T02:34:00,272 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/7ac40aeb8b8b45778453791b64bd3670, entries=150, sequenceid=105, filesize=11.7 K 2024-12-03T02:34:00,277 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=120.76 KB/123660 for 64446d8651566fb34f3be9578452136e in 578ms, sequenceid=105, compaction requested=true 2024-12-03T02:34:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-03T02:34:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-03T02:34:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:00,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T02:34:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:00,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:00,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-03T02:34:00,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3660 sec 2024-12-03T02:34:00,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.3750 sec 2024-12-03T02:34:00,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a14567124b00415096679991b17cea50_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193240268/Put/seqid=0 2024-12-03T02:34:00,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741960_1136 (size=19774) 2024-12-03T02:34:00,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193300313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193300314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:00,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193300424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:00,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193300424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:00,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193300634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:00,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193300634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:00,722 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:00,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a14567124b00415096679991b17cea50_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a14567124b00415096679991b17cea50_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:00,735 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/5734bd349f2d4057908dd3c20a70d3e5, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:00,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/5734bd349f2d4057908dd3c20a70d3e5 is 175, key is test_row_0/A:col10/1733193240268/Put/seqid=0 2024-12-03T02:34:00,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741961_1137 (size=57033) 2024-12-03T02:34:00,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193300945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193300946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:00,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T02:34:01,064 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:01,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-03T02:34:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T02:34:01,067 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:01,067 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:01,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:01,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/5734bd349f2d4057908dd3c20a70d3e5 2024-12-03T02:34:01,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/d3b1b198dff34931b6948d506de08a3e is 50, key is test_row_0/B:col10/1733193240268/Put/seqid=0 2024-12-03T02:34:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T02:34:01,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741962_1138 (size=12151) 2024-12-03T02:34:01,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/d3b1b198dff34931b6948d506de08a3e 2024-12-03T02:34:01,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/c42b0059f8d84d6c84f4898ea0cd4288 is 50, key is test_row_0/C:col10/1733193240268/Put/seqid=0 2024-12-03T02:34:01,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741963_1139 (size=12151) 2024-12-03T02:34:01,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T02:34:01,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:01,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:01,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:01,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T02:34:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:01,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T02:34:01,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193301455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:01,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193301455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:01,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T02:34:01,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:01,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:01,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:01,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/c42b0059f8d84d6c84f4898ea0cd4288 2024-12-03T02:34:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/5734bd349f2d4057908dd3c20a70d3e5 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5734bd349f2d4057908dd3c20a70d3e5 2024-12-03T02:34:01,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5734bd349f2d4057908dd3c20a70d3e5, entries=300, sequenceid=141, filesize=55.7 K 2024-12-03T02:34:01,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/d3b1b198dff34931b6948d506de08a3e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d3b1b198dff34931b6948d506de08a3e 2024-12-03T02:34:01,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d3b1b198dff34931b6948d506de08a3e, entries=150, sequenceid=141, filesize=11.9 K 2024-12-03T02:34:01,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/c42b0059f8d84d6c84f4898ea0cd4288 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/c42b0059f8d84d6c84f4898ea0cd4288 2024-12-03T02:34:01,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/c42b0059f8d84d6c84f4898ea0cd4288, entries=150, sequenceid=141, filesize=11.9 K 2024-12-03T02:34:01,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 64446d8651566fb34f3be9578452136e in 1355ms, sequenceid=141, compaction requested=true 2024-12-03T02:34:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:01,634 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:01,634 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:01,636 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:01,636 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/B is initiating minor compaction (all files) 2024-12-03T02:34:01,636 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/B in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,636 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/52465c021d1946aa8478911ace9097cb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e935f79f20ce4f87a1ab42de3b17697d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5b2652c21d3f4607a6b48eb14b920006, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d3b1b198dff34931b6948d506de08a3e] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=47.1 K 2024-12-03T02:34:01,637 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158595 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:01,637 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/A is initiating minor compaction (all files) 2024-12-03T02:34:01,637 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/A in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,637 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7d7d283607894ea2bca81790ce4d0283, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/fbe4a3024087488bae9ad17a656be48f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb6bc91600204400b5221afe46976ad6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5734bd349f2d4057908dd3c20a70d3e5] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=154.9 K 2024-12-03T02:34:01,637 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,637 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7d7d283607894ea2bca81790ce4d0283, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/fbe4a3024087488bae9ad17a656be48f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb6bc91600204400b5221afe46976ad6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5734bd349f2d4057908dd3c20a70d3e5] 2024-12-03T02:34:01,638 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 52465c021d1946aa8478911ace9097cb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733193237055 2024-12-03T02:34:01,638 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7d283607894ea2bca81790ce4d0283, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733193237055 2024-12-03T02:34:01,638 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe4a3024087488bae9ad17a656be48f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733193237788 2024-12-03T02:34:01,638 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e935f79f20ce4f87a1ab42de3b17697d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733193237788 2024-12-03T02:34:01,639 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb6bc91600204400b5221afe46976ad6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733193239004 2024-12-03T02:34:01,639 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b2652c21d3f4607a6b48eb14b920006, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733193239004 2024-12-03T02:34:01,639 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5734bd349f2d4057908dd3c20a70d3e5, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733193240246 2024-12-03T02:34:01,639 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b1b198dff34931b6948d506de08a3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733193240251 2024-12-03T02:34:01,646 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:01,647 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:01,648 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/047de2d5e90c4d4ca0642b03ebada8c9 is 50, key is test_row_0/B:col10/1733193240268/Put/seqid=0 2024-12-03T02:34:01,649 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203a1e4b9b2030d4b38be15cab71c2a681d_64446d8651566fb34f3be9578452136e store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:01,652 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203a1e4b9b2030d4b38be15cab71c2a681d_64446d8651566fb34f3be9578452136e, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:01,652 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a1e4b9b2030d4b38be15cab71c2a681d_64446d8651566fb34f3be9578452136e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:01,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741964_1140 (size=12391) 2024-12-03T02:34:01,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741965_1141 (size=4469) 2024-12-03T02:34:01,664 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#A#compaction#118 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:01,665 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/7f45ccd57f3946f7895e07c5aa5a45dc is 175, key is test_row_0/A:col10/1733193240268/Put/seqid=0 2024-12-03T02:34:01,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741966_1142 (size=31345) 2024-12-03T02:34:01,671 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/047de2d5e90c4d4ca0642b03ebada8c9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/047de2d5e90c4d4ca0642b03ebada8c9 2024-12-03T02:34:01,677 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64446d8651566fb34f3be9578452136e/B of 64446d8651566fb34f3be9578452136e into 047de2d5e90c4d4ca0642b03ebada8c9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:01,677 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:01,677 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/B, priority=12, startTime=1733193241634; duration=0sec 2024-12-03T02:34:01,677 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:01,677 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:B 2024-12-03T02:34:01,677 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:01,678 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/7f45ccd57f3946f7895e07c5aa5a45dc as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7f45ccd57f3946f7895e07c5aa5a45dc 2024-12-03T02:34:01,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T02:34:01,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,680 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T02:34:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:01,681 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:01,681 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/C is initiating minor compaction (all files) 2024-12-03T02:34:01,681 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/C in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:01,681 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/58c6024ad94147a2bb4ec4a96fbe3b3c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/19a0253660924708a37fa8e32055d2c1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/7ac40aeb8b8b45778453791b64bd3670, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/c42b0059f8d84d6c84f4898ea0cd4288] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=47.1 K 2024-12-03T02:34:01,682 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 58c6024ad94147a2bb4ec4a96fbe3b3c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733193237055 2024-12-03T02:34:01,683 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a0253660924708a37fa8e32055d2c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733193237788 2024-12-03T02:34:01,684 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac40aeb8b8b45778453791b64bd3670, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733193239004 2024-12-03T02:34:01,684 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting c42b0059f8d84d6c84f4898ea0cd4288, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733193240251 2024-12-03T02:34:01,688 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64446d8651566fb34f3be9578452136e/A of 64446d8651566fb34f3be9578452136e into 7f45ccd57f3946f7895e07c5aa5a45dc(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:01,688 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:01,688 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/A, priority=12, startTime=1733193241634; duration=0sec 2024-12-03T02:34:01,688 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:01,688 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:A 2024-12-03T02:34:01,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d2aee8ba38374ad8a5afbbbfab3128c3_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193240311/Put/seqid=0 2024-12-03T02:34:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T02:34:01,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741967_1143 (size=12304) 2024-12-03T02:34:01,718 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#C#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:01,718 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/a0d2bac3b1e94b8e9472c6446bba296e is 50, key is test_row_0/C:col10/1733193240268/Put/seqid=0 2024-12-03T02:34:01,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741968_1144 (size=12391) 2024-12-03T02:34:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:02,125 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d2aee8ba38374ad8a5afbbbfab3128c3_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d2aee8ba38374ad8a5afbbbfab3128c3_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:02,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/adbe6fc00ce049c5aadf7ebb753a245f, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:02,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/adbe6fc00ce049c5aadf7ebb753a245f is 175, key is test_row_0/A:col10/1733193240311/Put/seqid=0 2024-12-03T02:34:02,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741969_1145 (size=31105) 2024-12-03T02:34:02,135 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/adbe6fc00ce049c5aadf7ebb753a245f 2024-12-03T02:34:02,138 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/a0d2bac3b1e94b8e9472c6446bba296e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a0d2bac3b1e94b8e9472c6446bba296e 2024-12-03T02:34:02,145 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64446d8651566fb34f3be9578452136e/C of 64446d8651566fb34f3be9578452136e into a0d2bac3b1e94b8e9472c6446bba296e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:02,145 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:02,146 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/C, priority=12, startTime=1733193241634; duration=0sec 2024-12-03T02:34:02,146 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:02,146 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:C 2024-12-03T02:34:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/33100ce00a194fc7ad00c992beedd343 is 50, key is test_row_0/B:col10/1733193240311/Put/seqid=0 2024-12-03T02:34:02,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741970_1146 (size=12151) 2024-12-03T02:34:02,155 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/33100ce00a194fc7ad00c992beedd343 2024-12-03T02:34:02,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/96526ebad643412eb9f8d34793f9bf8b is 50, key is test_row_0/C:col10/1733193240311/Put/seqid=0 2024-12-03T02:34:02,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741971_1147 (size=12151) 2024-12-03T02:34:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T02:34:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:02,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:02,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193302533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:02,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193242482, max timestamp=1733193242535Num uniques 859; 2024-12-03T02:34:02,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:02,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193242482, max timestamp=1733193242535Num uniques 859; 2024-12-03T02:34:02,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:02,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193242482, max timestamp=1733193242535Num uniques 859; 2024-12-03T02:34:02,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:02,571 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/96526ebad643412eb9f8d34793f9bf8b 2024-12-03T02:34:02,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/adbe6fc00ce049c5aadf7ebb753a245f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/adbe6fc00ce049c5aadf7ebb753a245f 2024-12-03T02:34:02,591 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/adbe6fc00ce049c5aadf7ebb753a245f, entries=150, sequenceid=153, filesize=30.4 K 2024-12-03T02:34:02,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/33100ce00a194fc7ad00c992beedd343 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/33100ce00a194fc7ad00c992beedd343 2024-12-03T02:34:02,597 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/33100ce00a194fc7ad00c992beedd343, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T02:34:02,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/96526ebad643412eb9f8d34793f9bf8b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/96526ebad643412eb9f8d34793f9bf8b 2024-12-03T02:34:02,604 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/96526ebad643412eb9f8d34793f9bf8b, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T02:34:02,605 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=114.05 KB/116790 for 64446d8651566fb34f3be9578452136e in 925ms, sequenceid=153, compaction requested=false 2024-12-03T02:34:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-03T02:34:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-03T02:34:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:02,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-12-03T02:34:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:02,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-03T02:34:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:02,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:02,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5380 sec 2024-12-03T02:34:02,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.5450 sec 2024-12-03T02:34:02,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038a0f2ce1ac88404dbe798884919c618e_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193242579/Put/seqid=0 2024-12-03T02:34:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741972_1148 (size=14794) 2024-12-03T02:34:02,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:02,649 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038a0f2ce1ac88404dbe798884919c618e_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038a0f2ce1ac88404dbe798884919c618e_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:02,650 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/75876d2b9f9647a1a6212f65e9286696, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:02,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/75876d2b9f9647a1a6212f65e9286696 is 175, key is test_row_0/A:col10/1733193242579/Put/seqid=0 2024-12-03T02:34:02,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193302661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193302661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:02,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741973_1149 (size=39749) 2024-12-03T02:34:02,683 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/75876d2b9f9647a1a6212f65e9286696 2024-12-03T02:34:02,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/09fa679c4e3d4b37b134ebb9045fb46a is 50, key is test_row_0/B:col10/1733193242579/Put/seqid=0 2024-12-03T02:34:02,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741974_1150 (size=12151) 2024-12-03T02:34:02,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193302775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193302775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193302984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193302985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193303105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:03,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/09fa679c4e3d4b37b134ebb9045fb46a 2024-12-03T02:34:03,113 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:03,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/46af28506be24beebcf763a17f344ff6 is 50, key is test_row_0/C:col10/1733193242579/Put/seqid=0 2024-12-03T02:34:03,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193303135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:03,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193303135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:03,139 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:03,143 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:03,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:03,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741975_1151 (size=12151) 2024-12-03T02:34:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T02:34:03,215 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:03,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-03T02:34:03,217 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T02:34:03,218 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:03,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193303295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:03,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193303295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T02:34:03,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T02:34:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:03,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:03,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T02:34:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:03,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:03,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T02:34:03,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/46af28506be24beebcf763a17f344ff6 2024-12-03T02:34:03,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/75876d2b9f9647a1a6212f65e9286696 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/75876d2b9f9647a1a6212f65e9286696 2024-12-03T02:34:03,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/75876d2b9f9647a1a6212f65e9286696, entries=200, sequenceid=188, filesize=38.8 K 2024-12-03T02:34:03,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/09fa679c4e3d4b37b134ebb9045fb46a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/09fa679c4e3d4b37b134ebb9045fb46a 2024-12-03T02:34:03,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/09fa679c4e3d4b37b134ebb9045fb46a, entries=150, sequenceid=188, filesize=11.9 K 2024-12-03T02:34:03,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/46af28506be24beebcf763a17f344ff6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/46af28506be24beebcf763a17f344ff6 2024-12-03T02:34:03,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/46af28506be24beebcf763a17f344ff6, entries=150, sequenceid=188, filesize=11.9 K 2024-12-03T02:34:03,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=60.38 KB/61830 for 64446d8651566fb34f3be9578452136e in 982ms, sequenceid=188, compaction requested=true 2024-12-03T02:34:03,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:03,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:03,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:03,591 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:03,591 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:03,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:03,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:03,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:03,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:03,592 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:03,592 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:03,593 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/A is initiating minor compaction (all files) 2024-12-03T02:34:03,593 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/B is initiating minor compaction (all files) 2024-12-03T02:34:03,593 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/A in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:03,593 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/B in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:03,593 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7f45ccd57f3946f7895e07c5aa5a45dc, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/adbe6fc00ce049c5aadf7ebb753a245f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/75876d2b9f9647a1a6212f65e9286696] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=99.8 K 2024-12-03T02:34:03,593 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/047de2d5e90c4d4ca0642b03ebada8c9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/33100ce00a194fc7ad00c992beedd343, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/09fa679c4e3d4b37b134ebb9045fb46a] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=35.8 K 2024-12-03T02:34:03,593 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:03,593 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7f45ccd57f3946f7895e07c5aa5a45dc, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/adbe6fc00ce049c5aadf7ebb753a245f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/75876d2b9f9647a1a6212f65e9286696] 2024-12-03T02:34:03,593 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f45ccd57f3946f7895e07c5aa5a45dc, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733193240251 2024-12-03T02:34:03,593 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 047de2d5e90c4d4ca0642b03ebada8c9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733193240251 2024-12-03T02:34:03,594 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting adbe6fc00ce049c5aadf7ebb753a245f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733193240306 2024-12-03T02:34:03,594 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 33100ce00a194fc7ad00c992beedd343, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733193240306 2024-12-03T02:34:03,594 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75876d2b9f9647a1a6212f65e9286696, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733193242564 2024-12-03T02:34:03,594 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fa679c4e3d4b37b134ebb9045fb46a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733193242564 2024-12-03T02:34:03,600 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:03,602 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#B#compaction#127 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:03,603 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/2c0c8395773b45c38c4a2577dd3a578e is 50, key is test_row_0/B:col10/1733193242579/Put/seqid=0 2024-12-03T02:34:03,604 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120340825d0cf4a04e10a92b1ed99c239c4f_64446d8651566fb34f3be9578452136e store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:03,606 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120340825d0cf4a04e10a92b1ed99c239c4f_64446d8651566fb34f3be9578452136e, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:03,606 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120340825d0cf4a04e10a92b1ed99c239c4f_64446d8651566fb34f3be9578452136e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:03,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741977_1153 (size=4469) 2024-12-03T02:34:03,624 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#A#compaction#126 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:03,625 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/e5ae99affd3a4bc7809bfa9f2499a154 is 175, key is test_row_0/A:col10/1733193242579/Put/seqid=0 2024-12-03T02:34:03,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741976_1152 (size=12493) 2024-12-03T02:34:03,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741978_1154 (size=31447) 2024-12-03T02:34:03,635 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/2c0c8395773b45c38c4a2577dd3a578e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/2c0c8395773b45c38c4a2577dd3a578e 2024-12-03T02:34:03,641 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/B of 64446d8651566fb34f3be9578452136e into 2c0c8395773b45c38c4a2577dd3a578e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:03,641 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:03,641 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/B, priority=13, startTime=1733193243591; duration=0sec 2024-12-03T02:34:03,641 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:03,641 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:B 2024-12-03T02:34:03,641 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:03,642 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:03,642 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/C is initiating minor compaction (all files) 2024-12-03T02:34:03,643 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/C in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:03,643 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a0d2bac3b1e94b8e9472c6446bba296e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/96526ebad643412eb9f8d34793f9bf8b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/46af28506be24beebcf763a17f344ff6] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=35.8 K 2024-12-03T02:34:03,643 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d2bac3b1e94b8e9472c6446bba296e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733193240251 2024-12-03T02:34:03,644 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 96526ebad643412eb9f8d34793f9bf8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733193240306 2024-12-03T02:34:03,644 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 46af28506be24beebcf763a17f344ff6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733193242564 2024-12-03T02:34:03,651 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#C#compaction#128 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:03,651 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/bec5a9a0bbd843d3ae7d5e2f5c3cc948 is 50, key is test_row_0/C:col10/1733193242579/Put/seqid=0 2024-12-03T02:34:03,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741979_1155 (size=12493) 2024-12-03T02:34:03,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T02:34:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:03,677 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T02:34:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:03,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120394f104b5556b4ead9978cad6cd93fdad_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193242641/Put/seqid=0 2024-12-03T02:34:03,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741980_1156 (size=12304) 2024-12-03T02:34:03,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:03,697 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120394f104b5556b4ead9978cad6cd93fdad_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120394f104b5556b4ead9978cad6cd93fdad_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:03,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/daea7e744c64424dadeacdf6d8fd9eb0, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:03,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/daea7e744c64424dadeacdf6d8fd9eb0 is 175, key is test_row_0/A:col10/1733193242641/Put/seqid=0 2024-12-03T02:34:03,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741981_1157 (size=31105) 2024-12-03T02:34:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:03,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T02:34:03,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193303843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:03,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193303845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:03,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193303955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193303955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:04,042 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/e5ae99affd3a4bc7809bfa9f2499a154 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/e5ae99affd3a4bc7809bfa9f2499a154 2024-12-03T02:34:04,047 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/A of 64446d8651566fb34f3be9578452136e into e5ae99affd3a4bc7809bfa9f2499a154(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:04,048 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:04,048 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/A, priority=13, startTime=1733193243591; duration=0sec 2024-12-03T02:34:04,048 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:04,048 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:A 2024-12-03T02:34:04,066 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/bec5a9a0bbd843d3ae7d5e2f5c3cc948 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bec5a9a0bbd843d3ae7d5e2f5c3cc948 2024-12-03T02:34:04,083 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/C of 64446d8651566fb34f3be9578452136e into bec5a9a0bbd843d3ae7d5e2f5c3cc948(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:04,084 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:04,084 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/C, priority=13, startTime=1733193243591; duration=0sec 2024-12-03T02:34:04,084 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:04,084 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:C 2024-12-03T02:34:04,103 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/daea7e744c64424dadeacdf6d8fd9eb0 2024-12-03T02:34:04,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/f8be8a9ad32548deae2bd091a4a9e501 is 50, key is test_row_0/B:col10/1733193242641/Put/seqid=0 2024-12-03T02:34:04,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741982_1158 (size=12151) 2024-12-03T02:34:04,139 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/f8be8a9ad32548deae2bd091a4a9e501 2024-12-03T02:34:04,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/49df7ffc273748acbdf0c9b74babee45 is 50, key is test_row_0/C:col10/1733193242641/Put/seqid=0 2024-12-03T02:34:04,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:04,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193304165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193304165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:04,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741983_1159 (size=12151) 2024-12-03T02:34:04,179 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/49df7ffc273748acbdf0c9b74babee45 2024-12-03T02:34:04,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/daea7e744c64424dadeacdf6d8fd9eb0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/daea7e744c64424dadeacdf6d8fd9eb0 2024-12-03T02:34:04,195 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/daea7e744c64424dadeacdf6d8fd9eb0, entries=150, sequenceid=201, filesize=30.4 K 2024-12-03T02:34:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/f8be8a9ad32548deae2bd091a4a9e501 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/f8be8a9ad32548deae2bd091a4a9e501 2024-12-03T02:34:04,204 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/f8be8a9ad32548deae2bd091a4a9e501, entries=150, sequenceid=201, filesize=11.9 K 2024-12-03T02:34:04,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/49df7ffc273748acbdf0c9b74babee45 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/49df7ffc273748acbdf0c9b74babee45 2024-12-03T02:34:04,215 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/49df7ffc273748acbdf0c9b74babee45, entries=150, sequenceid=201, filesize=11.9 K 2024-12-03T02:34:04,216 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 64446d8651566fb34f3be9578452136e in 539ms, sequenceid=201, compaction requested=false 2024-12-03T02:34:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-03T02:34:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-03T02:34:04,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-03T02:34:04,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-12-03T02:34:04,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.0100 sec 2024-12-03T02:34:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T02:34:04,355 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:04,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-03T02:34:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T02:34:04,356 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:04,357 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:04,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T02:34:04,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T02:34:04,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:04,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:04,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:04,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:04,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:04,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:04,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120391d55c16e30d4e74ad2fe4ef130593b2_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193243843/Put/seqid=0 2024-12-03T02:34:04,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741984_1160 (size=14794) 2024-12-03T02:34:04,501 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:04,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:04,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193304502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193304503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:04,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T02:34:04,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:04,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120391d55c16e30d4e74ad2fe4ef130593b2_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120391d55c16e30d4e74ad2fe4ef130593b2_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:04,517 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/5ce16ddd849c419696911e34841e947d, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:04,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/5ce16ddd849c419696911e34841e947d is 175, key is test_row_0/A:col10/1733193243843/Put/seqid=0 2024-12-03T02:34:04,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741985_1161 (size=39749) 2024-12-03T02:34:04,524 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/5ce16ddd849c419696911e34841e947d 2024-12-03T02:34:04,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/bcea4cd7fdc446e09b7cb9fad36f880e is 50, key is test_row_0/B:col10/1733193243843/Put/seqid=0 2024-12-03T02:34:04,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741986_1162 (size=12151) 2024-12-03T02:34:04,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/bcea4cd7fdc446e09b7cb9fad36f880e 2024-12-03T02:34:04,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/21537562ca2048cbb8d66a6cb5fc2249 is 50, key is test_row_0/C:col10/1733193243843/Put/seqid=0 2024-12-03T02:34:04,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741987_1163 (size=12151) 2024-12-03T02:34:04,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193304614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193304614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:04,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T02:34:04,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:04,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T02:34:04,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T02:34:04,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:04,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193304824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193304824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:04,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/21537562ca2048cbb8d66a6cb5fc2249 2024-12-03T02:34:04,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/5ce16ddd849c419696911e34841e947d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5ce16ddd849c419696911e34841e947d 2024-12-03T02:34:04,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5ce16ddd849c419696911e34841e947d, entries=200, sequenceid=224, filesize=38.8 K 2024-12-03T02:34:04,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/bcea4cd7fdc446e09b7cb9fad36f880e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/bcea4cd7fdc446e09b7cb9fad36f880e 2024-12-03T02:34:04,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T02:34:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:04,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/bcea4cd7fdc446e09b7cb9fad36f880e, entries=150, sequenceid=224, filesize=11.9 K 2024-12-03T02:34:04,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/21537562ca2048cbb8d66a6cb5fc2249 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/21537562ca2048cbb8d66a6cb5fc2249 2024-12-03T02:34:04,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/21537562ca2048cbb8d66a6cb5fc2249, entries=150, sequenceid=224, filesize=11.9 K 2024-12-03T02:34:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T02:34:04,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 64446d8651566fb34f3be9578452136e in 511ms, sequenceid=224, compaction requested=true 2024-12-03T02:34:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:04,990 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:04,990 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:04,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:04,991 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:04,991 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/B is initiating minor compaction (all files) 2024-12-03T02:34:04,991 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/B in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,991 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:04,991 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/2c0c8395773b45c38c4a2577dd3a578e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/f8be8a9ad32548deae2bd091a4a9e501, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/bcea4cd7fdc446e09b7cb9fad36f880e] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=35.9 K 2024-12-03T02:34:04,991 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/A is initiating minor compaction (all files) 2024-12-03T02:34:04,992 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/A in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,992 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/e5ae99affd3a4bc7809bfa9f2499a154, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/daea7e744c64424dadeacdf6d8fd9eb0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5ce16ddd849c419696911e34841e947d] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=99.9 K 2024-12-03T02:34:04,992 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:04,992 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/e5ae99affd3a4bc7809bfa9f2499a154, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/daea7e744c64424dadeacdf6d8fd9eb0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5ce16ddd849c419696911e34841e947d] 2024-12-03T02:34:04,992 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c0c8395773b45c38c4a2577dd3a578e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733193242564 2024-12-03T02:34:04,993 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting f8be8a9ad32548deae2bd091a4a9e501, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733193242635 2024-12-03T02:34:04,993 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ae99affd3a4bc7809bfa9f2499a154, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733193242564 2024-12-03T02:34:04,994 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting daea7e744c64424dadeacdf6d8fd9eb0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733193242635 2024-12-03T02:34:04,994 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting bcea4cd7fdc446e09b7cb9fad36f880e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733193243841 2024-12-03T02:34:04,994 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ce16ddd849c419696911e34841e947d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733193243826 2024-12-03T02:34:05,008 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:05,008 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:05,009 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/5204aed711054587a296cf765f0d7167 is 50, key is test_row_0/B:col10/1733193243843/Put/seqid=0 2024-12-03T02:34:05,023 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120398a359e991064b288647e3c9d6b2f1b2_64446d8651566fb34f3be9578452136e store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:05,025 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120398a359e991064b288647e3c9d6b2f1b2_64446d8651566fb34f3be9578452136e, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:05,025 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120398a359e991064b288647e3c9d6b2f1b2_64446d8651566fb34f3be9578452136e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:05,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741988_1164 (size=12595) 2024-12-03T02:34:05,064 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/5204aed711054587a296cf765f0d7167 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5204aed711054587a296cf765f0d7167 2024-12-03T02:34:05,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741989_1165 (size=4469) 2024-12-03T02:34:05,075 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#A#compaction#136 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:05,076 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/27940a1da0ac474ea56b635d7fa28dfe is 175, key is test_row_0/A:col10/1733193243843/Put/seqid=0 2024-12-03T02:34:05,083 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/B of 64446d8651566fb34f3be9578452136e into 5204aed711054587a296cf765f0d7167(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:05,083 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:05,083 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/B, priority=13, startTime=1733193244990; duration=0sec 2024-12-03T02:34:05,083 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:05,083 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:B 2024-12-03T02:34:05,084 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:05,085 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:05,085 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/C is initiating minor compaction (all files) 2024-12-03T02:34:05,085 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/C in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:05,086 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bec5a9a0bbd843d3ae7d5e2f5c3cc948, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/49df7ffc273748acbdf0c9b74babee45, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/21537562ca2048cbb8d66a6cb5fc2249] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=35.9 K 2024-12-03T02:34:05,087 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting bec5a9a0bbd843d3ae7d5e2f5c3cc948, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733193242564 2024-12-03T02:34:05,088 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 49df7ffc273748acbdf0c9b74babee45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733193242635 2024-12-03T02:34:05,088 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 21537562ca2048cbb8d66a6cb5fc2249, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733193243841 2024-12-03T02:34:05,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741990_1166 (size=31549) 2024-12-03T02:34:05,103 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#C#compaction#137 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:05,103 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/08d22670842441079c78c52083081231 is 50, key is test_row_0/C:col10/1733193243843/Put/seqid=0 2024-12-03T02:34:05,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T02:34:05,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:05,127 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T02:34:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:05,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:05,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741991_1167 (size=12595) 2024-12-03T02:34:05,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:05,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203afde2d5bb9fd4cb283ba6f111418058e_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193244501/Put/seqid=0 2024-12-03T02:34:05,148 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/08d22670842441079c78c52083081231 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/08d22670842441079c78c52083081231 2024-12-03T02:34:05,158 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/C of 64446d8651566fb34f3be9578452136e into 08d22670842441079c78c52083081231(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:05,158 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:05,158 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/C, priority=13, startTime=1733193244990; duration=0sec 2024-12-03T02:34:05,158 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:05,158 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:C 2024-12-03T02:34:05,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741992_1168 (size=12304) 2024-12-03T02:34:05,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193245178, max timestamp=1733193245194Num uniques 859; 2024-12-03T02:34:05,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:05,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193245178, max timestamp=1733193245194Num uniques 859; 2024-12-03T02:34:05,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:05,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193245178, max timestamp=1733193245194Num uniques 859; 2024-12-03T02:34:05,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:05,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193305264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:05,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193305266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:05,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193305375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:05,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193305384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T02:34:05,499 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/27940a1da0ac474ea56b635d7fa28dfe as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/27940a1da0ac474ea56b635d7fa28dfe 2024-12-03T02:34:05,504 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/A of 64446d8651566fb34f3be9578452136e into 27940a1da0ac474ea56b635d7fa28dfe(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:05,504 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:05,504 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/A, priority=13, startTime=1733193244990; duration=0sec 2024-12-03T02:34:05,504 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:05,504 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:A 2024-12-03T02:34:05,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:05,568 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203afde2d5bb9fd4cb283ba6f111418058e_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203afde2d5bb9fd4cb283ba6f111418058e_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:05,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/727746d6cee34fc1b0050f18e73be44a, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:05,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/727746d6cee34fc1b0050f18e73be44a is 175, key is test_row_0/A:col10/1733193244501/Put/seqid=0 2024-12-03T02:34:05,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741993_1169 (size=31105) 2024-12-03T02:34:05,585 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/727746d6cee34fc1b0050f18e73be44a 2024-12-03T02:34:05,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193305585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:05,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/9b967a054ad44a5e894d466f4519de9f is 50, key is test_row_0/B:col10/1733193244501/Put/seqid=0 2024-12-03T02:34:05,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741994_1170 (size=12151) 2024-12-03T02:34:05,599 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/9b967a054ad44a5e894d466f4519de9f 2024-12-03T02:34:05,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193305599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:05,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/db59487f536b4fb88152f6d94a898893 is 50, key is test_row_0/C:col10/1733193244501/Put/seqid=0 2024-12-03T02:34:05,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741995_1171 (size=12151) 2024-12-03T02:34:05,624 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/db59487f536b4fb88152f6d94a898893 2024-12-03T02:34:05,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/727746d6cee34fc1b0050f18e73be44a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/727746d6cee34fc1b0050f18e73be44a 2024-12-03T02:34:05,639 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/727746d6cee34fc1b0050f18e73be44a, entries=150, sequenceid=236, filesize=30.4 K 2024-12-03T02:34:05,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/9b967a054ad44a5e894d466f4519de9f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/9b967a054ad44a5e894d466f4519de9f 2024-12-03T02:34:05,649 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/9b967a054ad44a5e894d466f4519de9f, entries=150, sequenceid=236, filesize=11.9 K 2024-12-03T02:34:05,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/db59487f536b4fb88152f6d94a898893 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/db59487f536b4fb88152f6d94a898893 2024-12-03T02:34:05,663 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/db59487f536b4fb88152f6d94a898893, entries=150, sequenceid=236, filesize=11.9 K 2024-12-03T02:34:05,669 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 64446d8651566fb34f3be9578452136e in 541ms, sequenceid=236, compaction requested=false 2024-12-03T02:34:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-03T02:34:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-03T02:34:05,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-03T02:34:05,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3180 sec 2024-12-03T02:34:05,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.3340 sec 2024-12-03T02:34:05,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-03T02:34:05,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:05,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:05,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:05,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:05,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:05,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:05,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039d567313692b45a98d62f4088272092f_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193245898/Put/seqid=0 2024-12-03T02:34:05,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193305926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:05,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193305926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:05,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:05,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:05,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741996_1172 (size=12454) 2024-12-03T02:34:05,941 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:05,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039d567313692b45a98d62f4088272092f_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039d567313692b45a98d62f4088272092f_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:05,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/2683d48b35644077bb175e5c707731e8, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:05,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/2683d48b35644077bb175e5c707731e8 is 175, key is test_row_0/A:col10/1733193245898/Put/seqid=0 2024-12-03T02:34:06,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741997_1173 (size=31255) 2024-12-03T02:34:06,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193306035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:06,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193306035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:06,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193306245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:06,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193306245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:06,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/2683d48b35644077bb175e5c707731e8 2024-12-03T02:34:06,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/d77771546ed2429e811311355a589bff is 50, key is test_row_0/B:col10/1733193245898/Put/seqid=0 2024-12-03T02:34:06,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741998_1174 (size=12301) 2024-12-03T02:34:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T02:34:06,505 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:06,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-03T02:34:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T02:34:06,507 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:06,507 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:06,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:06,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193306555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193306555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:06,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:06,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:06,586 INFO [master/64035cbb0b46:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T02:34:06,586 INFO [master/64035cbb0b46:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T02:34:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T02:34:06,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T02:34:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:06,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:06,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T02:34:06,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:06,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:06,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T02:34:06,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/d77771546ed2429e811311355a589bff 2024-12-03T02:34:06,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/a98ec502868f4f1395d41ab766740018 is 50, key is test_row_0/C:col10/1733193245898/Put/seqid=0 2024-12-03T02:34:06,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741999_1175 (size=12301) 2024-12-03T02:34:06,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T02:34:06,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:06,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:06,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:06,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:06,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:07,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193307067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:07,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:07,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193307079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:07,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:07,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:07,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T02:34:07,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T02:34:07,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:07,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:07,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:07,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:07,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:07,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/a98ec502868f4f1395d41ab766740018 2024-12-03T02:34:07,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T02:34:07,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:07,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:07,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:07,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:07,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:07,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/2683d48b35644077bb175e5c707731e8 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/2683d48b35644077bb175e5c707731e8 2024-12-03T02:34:07,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/2683d48b35644077bb175e5c707731e8, entries=150, sequenceid=273, filesize=30.5 K 2024-12-03T02:34:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/d77771546ed2429e811311355a589bff as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d77771546ed2429e811311355a589bff 2024-12-03T02:34:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d77771546ed2429e811311355a589bff, entries=150, sequenceid=273, filesize=12.0 K 2024-12-03T02:34:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/a98ec502868f4f1395d41ab766740018 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a98ec502868f4f1395d41ab766740018 2024-12-03T02:34:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a98ec502868f4f1395d41ab766740018, entries=150, sequenceid=273, filesize=12.0 K 2024-12-03T02:34:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 64446d8651566fb34f3be9578452136e in 1544ms, sequenceid=273, compaction requested=true 2024-12-03T02:34:07,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:34:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T02:34:07,444 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:07,444 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,456 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93909 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:07,456 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/A is initiating minor compaction (all files) 2024-12-03T02:34:07,456 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/A in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,456 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/27940a1da0ac474ea56b635d7fa28dfe, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/727746d6cee34fc1b0050f18e73be44a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/2683d48b35644077bb175e5c707731e8] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=91.7 K 2024-12-03T02:34:07,456 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:07,456 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/27940a1da0ac474ea56b635d7fa28dfe, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/727746d6cee34fc1b0050f18e73be44a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/2683d48b35644077bb175e5c707731e8] 2024-12-03T02:34:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,459 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:07,459 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/C is initiating minor compaction (all files) 2024-12-03T02:34:07,460 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/C in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,460 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/08d22670842441079c78c52083081231, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/db59487f536b4fb88152f6d94a898893, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a98ec502868f4f1395d41ab766740018] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=36.2 K 2024-12-03T02:34:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T02:34:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:07,467 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 27940a1da0ac474ea56b635d7fa28dfe, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733193243841 2024-12-03T02:34:07,467 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08d22670842441079c78c52083081231, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733193243841 2024-12-03T02:34:07,467 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:34:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:07,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,475 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting db59487f536b4fb88152f6d94a898893, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733193244491 2024-12-03T02:34:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,476 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 727746d6cee34fc1b0050f18e73be44a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733193244491 2024-12-03T02:34:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,480 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2683d48b35644077bb175e5c707731e8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733193245255 2024-12-03T02:34:07,480 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a98ec502868f4f1395d41ab766740018, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733193245255 2024-12-03T02:34:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c24d82fd84f149cb9b27b667e07d18c4_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193245914/Put/seqid=0 2024-12-03T02:34:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,576 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,590 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#C#compaction#146 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,591 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/75939ccaea024ca9afe5f31f7591486d is 50, key is test_row_0/C:col10/1733193245898/Put/seqid=0 2024-12-03T02:34:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742000_1176 (size=9914) 2024-12-03T02:34:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,605 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033532614715ea427fb1bbe91d59222076_64446d8651566fb34f3be9578452136e store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,607 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033532614715ea427fb1bbe91d59222076_64446d8651566fb34f3be9578452136e, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:07,607 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033532614715ea427fb1bbe91d59222076_64446d8651566fb34f3be9578452136e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T02:34:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742001_1177 (size=12847) 2024-12-03T02:34:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742002_1178 (size=4469) 2024-12-03T02:34:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,716 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#A#compaction#145 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:07,717 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/c968b69e8fbe45bfbcfcf537c72d7d28 is 175, key is test_row_0/A:col10/1733193245898/Put/seqid=0 2024-12-03T02:34:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742003_1179 (size=31801) 2024-12-03T02:34:07,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:07,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,025 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c24d82fd84f149cb9b27b667e07d18c4_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c24d82fd84f149cb9b27b667e07d18c4_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/3fd7f7e94adc4a6d80b983c22e00c8f9, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:08,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/3fd7f7e94adc4a6d80b983c22e00c8f9 is 175, key is test_row_0/A:col10/1733193245914/Put/seqid=0 2024-12-03T02:34:08,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,085 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/75939ccaea024ca9afe5f31f7591486d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/75939ccaea024ca9afe5f31f7591486d 2024-12-03T02:34:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742004_1180 (size=22561) 2024-12-03T02:34:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:08,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:08,098 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/3fd7f7e94adc4a6d80b983c22e00c8f9 2024-12-03T02:34:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,140 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/C of 64446d8651566fb34f3be9578452136e into 75939ccaea024ca9afe5f31f7591486d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:08,140 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:08,140 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/C, priority=13, startTime=1733193247444; duration=0sec 2024-12-03T02:34:08,140 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:08,140 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:C 2024-12-03T02:34:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,148 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,150 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:08,150 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/B is initiating minor compaction (all files) 2024-12-03T02:34:08,150 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/B in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,150 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5204aed711054587a296cf765f0d7167, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/9b967a054ad44a5e894d466f4519de9f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d77771546ed2429e811311355a589bff] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=36.2 K 2024-12-03T02:34:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,153 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5204aed711054587a296cf765f0d7167, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733193243841 2024-12-03T02:34:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,153 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b967a054ad44a5e894d466f4519de9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733193244491 2024-12-03T02:34:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,163 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting d77771546ed2429e811311355a589bff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733193245255 2024-12-03T02:34:08,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/4156aa05691647dc85ad7ba69f474394 is 50, key is test_row_0/B:col10/1733193245914/Put/seqid=0 2024-12-03T02:34:08,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742005_1181 (size=9857) 2024-12-03T02:34:08,200 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/4156aa05691647dc85ad7ba69f474394 2024-12-03T02:34:08,222 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/c968b69e8fbe45bfbcfcf537c72d7d28 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/c968b69e8fbe45bfbcfcf537c72d7d28 2024-12-03T02:34:08,224 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#B#compaction#148 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:08,225 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/679e4790f1bd4b86a851b76cf303e413 is 50, key is test_row_0/B:col10/1733193245898/Put/seqid=0 2024-12-03T02:34:08,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/eaad8a75da9245b5ada9783c994e8fe7 is 50, key is test_row_0/C:col10/1733193245914/Put/seqid=0 2024-12-03T02:34:08,238 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/A of 64446d8651566fb34f3be9578452136e into c968b69e8fbe45bfbcfcf537c72d7d28(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:08,238 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:08,238 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/A, priority=13, startTime=1733193247443; duration=0sec 2024-12-03T02:34:08,238 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:08,238 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:A 2024-12-03T02:34:08,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193308252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:08,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193308254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:08,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193248215, max timestamp=1733193248249Num uniques 859; 2024-12-03T02:34:08,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193248215, max timestamp=1733193248249Num uniques 859; 2024-12-03T02:34:08,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:08,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193248215, max timestamp=1733193248249Num uniques 859; 2024-12-03T02:34:08,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:34:08,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742007_1183 (size=9857) 2024-12-03T02:34:08,272 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/eaad8a75da9245b5ada9783c994e8fe7 2024-12-03T02:34:08,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/3fd7f7e94adc4a6d80b983c22e00c8f9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/3fd7f7e94adc4a6d80b983c22e00c8f9 2024-12-03T02:34:08,291 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/3fd7f7e94adc4a6d80b983c22e00c8f9, entries=100, sequenceid=283, filesize=22.0 K 2024-12-03T02:34:08,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/4156aa05691647dc85ad7ba69f474394 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/4156aa05691647dc85ad7ba69f474394 2024-12-03T02:34:08,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742006_1182 (size=12847) 2024-12-03T02:34:08,304 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/4156aa05691647dc85ad7ba69f474394, entries=100, sequenceid=283, filesize=9.6 K 2024-12-03T02:34:08,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/eaad8a75da9245b5ada9783c994e8fe7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/eaad8a75da9245b5ada9783c994e8fe7 2024-12-03T02:34:08,313 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/679e4790f1bd4b86a851b76cf303e413 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/679e4790f1bd4b86a851b76cf303e413 2024-12-03T02:34:08,316 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/eaad8a75da9245b5ada9783c994e8fe7, entries=100, sequenceid=283, filesize=9.6 K 2024-12-03T02:34:08,319 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=33.15 KB/33945 for 64446d8651566fb34f3be9578452136e in 852ms, sequenceid=283, compaction requested=false 2024-12-03T02:34:08,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:08,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-03T02:34:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-03T02:34:08,322 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/B of 64446d8651566fb34f3be9578452136e into 679e4790f1bd4b86a851b76cf303e413(size=12.5 K), total size for store is 22.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:08,322 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:08,322 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/B, priority=13, startTime=1733193247444; duration=0sec 2024-12-03T02:34:08,322 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:08,322 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:B 2024-12-03T02:34:08,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-03T02:34:08,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-12-03T02:34:08,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8210 sec 2024-12-03T02:34:08,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=53.28 KB heapSize=145.01 KB 2024-12-03T02:34:08,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:08,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:08,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:08,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:08,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:08,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:08,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b7e0e724280c4af89bcfc509f74ff8a1_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193248386/Put/seqid=0 2024-12-03T02:34:08,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742008_1184 (size=14994) 2024-12-03T02:34:08,440 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:08,446 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b7e0e724280c4af89bcfc509f74ff8a1_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b7e0e724280c4af89bcfc509f74ff8a1_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:08,448 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/a7678328bd80468d98d8a77e24aa5b22, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:08,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/a7678328bd80468d98d8a77e24aa5b22 is 175, key is test_row_0/A:col10/1733193248386/Put/seqid=0 2024-12-03T02:34:08,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742009_1185 (size=39949) 2024-12-03T02:34:08,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193248437, max timestamp=1733193248473Num uniques 859; 2024-12-03T02:34:08,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:08,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193248437, max timestamp=1733193248473Num uniques 859; 2024-12-03T02:34:08,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:08,479 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=17.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/a7678328bd80468d98d8a77e24aa5b22 2024-12-03T02:34:08,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193248437, max timestamp=1733193248473Num uniques 859; 2024-12-03T02:34:08,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:08,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/317492a851d0455596e6e2e12da7ca5c is 50, key is test_row_0/B:col10/1733193248386/Put/seqid=0 2024-12-03T02:34:08,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742010_1186 (size=12301) 2024-12-03T02:34:08,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/317492a851d0455596e6e2e12da7ca5c 2024-12-03T02:34:08,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/f20b52ceafb64d8eabbbf8d2a16720fc is 50, key is test_row_0/C:col10/1733193248386/Put/seqid=0 2024-12-03T02:34:08,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742011_1187 (size=12301) 2024-12-03T02:34:08,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/f20b52ceafb64d8eabbbf8d2a16720fc 2024-12-03T02:34:08,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193308576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:08,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193308575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/a7678328bd80468d98d8a77e24aa5b22 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/a7678328bd80468d98d8a77e24aa5b22 2024-12-03T02:34:08,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/a7678328bd80468d98d8a77e24aa5b22, entries=200, sequenceid=311, filesize=39.0 K 2024-12-03T02:34:08,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/317492a851d0455596e6e2e12da7ca5c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/317492a851d0455596e6e2e12da7ca5c 2024-12-03T02:34:08,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/317492a851d0455596e6e2e12da7ca5c, entries=150, sequenceid=311, filesize=12.0 K 2024-12-03T02:34:08,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/f20b52ceafb64d8eabbbf8d2a16720fc as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/f20b52ceafb64d8eabbbf8d2a16720fc 2024-12-03T02:34:08,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/f20b52ceafb64d8eabbbf8d2a16720fc, entries=150, sequenceid=311, filesize=12.0 K 2024-12-03T02:34:08,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.28 KB/54555, heapSize ~144.96 KB/148440, currentSize=134.18 KB/137400 for 64446d8651566fb34f3be9578452136e in 238ms, sequenceid=311, compaction requested=true 2024-12-03T02:34:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:08,630 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:08,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:08,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:08,632 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:08,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:08,633 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:08,633 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:08,633 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/B is initiating minor compaction (all files) 2024-12-03T02:34:08,633 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/B in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,633 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/679e4790f1bd4b86a851b76cf303e413, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/4156aa05691647dc85ad7ba69f474394, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/317492a851d0455596e6e2e12da7ca5c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=34.2 K 2024-12-03T02:34:08,633 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/A is initiating minor compaction (all files) 2024-12-03T02:34:08,633 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/A in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,633 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/c968b69e8fbe45bfbcfcf537c72d7d28, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/3fd7f7e94adc4a6d80b983c22e00c8f9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/a7678328bd80468d98d8a77e24aa5b22] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=92.1 K 2024-12-03T02:34:08,633 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,633 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 679e4790f1bd4b86a851b76cf303e413, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733193245255 2024-12-03T02:34:08,633 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/c968b69e8fbe45bfbcfcf537c72d7d28, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/3fd7f7e94adc4a6d80b983c22e00c8f9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/a7678328bd80468d98d8a77e24aa5b22] 2024-12-03T02:34:08,634 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 4156aa05691647dc85ad7ba69f474394, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733193245914 2024-12-03T02:34:08,634 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting c968b69e8fbe45bfbcfcf537c72d7d28, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733193245255 2024-12-03T02:34:08,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:08,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:08,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:08,634 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 317492a851d0455596e6e2e12da7ca5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733193248375 2024-12-03T02:34:08,639 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fd7f7e94adc4a6d80b983c22e00c8f9, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733193245914 2024-12-03T02:34:08,639 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7678328bd80468d98d8a77e24aa5b22, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733193248215 2024-12-03T02:34:08,648 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:08,648 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/69ae1f3e0d494d20b4de2935d347dc4f is 50, key is test_row_0/B:col10/1733193248386/Put/seqid=0 2024-12-03T02:34:08,655 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T02:34:08,656 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:08,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-03T02:34:08,659 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T02:34:08,660 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:08,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:08,664 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033b0a1bb114674d53a8f6e18da02e1199_64446d8651566fb34f3be9578452136e store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:08,666 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033b0a1bb114674d53a8f6e18da02e1199_64446d8651566fb34f3be9578452136e, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:08,666 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033b0a1bb114674d53a8f6e18da02e1199_64446d8651566fb34f3be9578452136e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:08,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742012_1188 (size=12949) 2024-12-03T02:34:08,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742013_1189 (size=4469) 2024-12-03T02:34:08,697 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/69ae1f3e0d494d20b4de2935d347dc4f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/69ae1f3e0d494d20b4de2935d347dc4f 2024-12-03T02:34:08,712 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/B of 64446d8651566fb34f3be9578452136e into 69ae1f3e0d494d20b4de2935d347dc4f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:08,712 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:08,712 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/B, priority=13, startTime=1733193248632; duration=0sec 2024-12-03T02:34:08,713 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:08,713 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:B 2024-12-03T02:34:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:08,713 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:08,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-03T02:34:08,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:08,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:34:08,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:08,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:34:08,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:08,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:34:08,714 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:08,714 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/C is initiating minor compaction (all files) 2024-12-03T02:34:08,714 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/C in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,714 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/75939ccaea024ca9afe5f31f7591486d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/eaad8a75da9245b5ada9783c994e8fe7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/f20b52ceafb64d8eabbbf8d2a16720fc] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=34.2 K 2024-12-03T02:34:08,715 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 75939ccaea024ca9afe5f31f7591486d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733193245255 2024-12-03T02:34:08,715 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting eaad8a75da9245b5ada9783c994e8fe7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733193245914 2024-12-03T02:34:08,716 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting f20b52ceafb64d8eabbbf8d2a16720fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733193248375 2024-12-03T02:34:08,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035407f3db5f384ed99c6b4a531684a739_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193248695/Put/seqid=0 2024-12-03T02:34:08,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193308729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:08,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742014_1190 (size=17534) 2024-12-03T02:34:08,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193308731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:08,742 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:08,742 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/87f76f5a45054ae9a7e7c118aa6854b1 is 50, key is test_row_0/C:col10/1733193248386/Put/seqid=0 2024-12-03T02:34:08,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742015_1191 (size=12949) 2024-12-03T02:34:08,757 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/87f76f5a45054ae9a7e7c118aa6854b1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/87f76f5a45054ae9a7e7c118aa6854b1 2024-12-03T02:34:08,762 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/C of 64446d8651566fb34f3be9578452136e into 87f76f5a45054ae9a7e7c118aa6854b1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:08,762 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:08,762 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/C, priority=13, startTime=1733193248634; duration=0sec 2024-12-03T02:34:08,762 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:08,762 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:C 2024-12-03T02:34:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T02:34:08,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T02:34:08,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:08,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:08,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:08,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193308834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:08,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:08,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193308857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:08,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:08,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:08,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T02:34:08,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:08,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:08,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:08,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T02:34:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193309048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:09,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193309065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:09,101 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#A#compaction#154 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:09,102 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/719ea14418834b3e92db927fc72dea9c is 175, key is test_row_0/A:col10/1733193248386/Put/seqid=0 2024-12-03T02:34:09,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T02:34:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:09,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,143 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742016_1192 (size=31903) 2024-12-03T02:34:09,164 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035407f3db5f384ed99c6b4a531684a739_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035407f3db5f384ed99c6b4a531684a739_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:09,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/d3d9cd166a244907a9efb02451699753, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:09,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/d3d9cd166a244907a9efb02451699753 is 175, key is test_row_0/A:col10/1733193248695/Put/seqid=0 2024-12-03T02:34:09,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742017_1193 (size=48639) 2024-12-03T02:34:09,224 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/d3d9cd166a244907a9efb02451699753 2024-12-03T02:34:09,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/ffb4739e76884fb89ee81a7b2af78420 is 50, key is test_row_0/B:col10/1733193248695/Put/seqid=0 2024-12-03T02:34:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T02:34:09,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T02:34:09,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:09,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:09,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:09,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742018_1194 (size=12301) 2024-12-03T02:34:09,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/ffb4739e76884fb89ee81a7b2af78420 2024-12-03T02:34:09,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/54c1f4886ba543ee9a0ee6949c923a94 is 50, key is test_row_0/C:col10/1733193248695/Put/seqid=0 2024-12-03T02:34:09,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193309355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:09,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742019_1195 (size=12301) 2024-12-03T02:34:09,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193309384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:09,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/54c1f4886ba543ee9a0ee6949c923a94 2024-12-03T02:34:09,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/d3d9cd166a244907a9efb02451699753 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d3d9cd166a244907a9efb02451699753 2024-12-03T02:34:09,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d3d9cd166a244907a9efb02451699753, entries=250, sequenceid=350, filesize=47.5 K 2024-12-03T02:34:09,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/ffb4739e76884fb89ee81a7b2af78420 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/ffb4739e76884fb89ee81a7b2af78420 2024-12-03T02:34:09,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T02:34:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:09,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:09,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/ffb4739e76884fb89ee81a7b2af78420, entries=150, sequenceid=350, filesize=12.0 K 2024-12-03T02:34:09,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/54c1f4886ba543ee9a0ee6949c923a94 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/54c1f4886ba543ee9a0ee6949c923a94 2024-12-03T02:34:09,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/54c1f4886ba543ee9a0ee6949c923a94, entries=150, sequenceid=350, filesize=12.0 K 2024-12-03T02:34:09,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 64446d8651566fb34f3be9578452136e in 794ms, sequenceid=350, compaction requested=false 2024-12-03T02:34:09,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:09,551 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/719ea14418834b3e92db927fc72dea9c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/719ea14418834b3e92db927fc72dea9c 2024-12-03T02:34:09,564 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/A of 64446d8651566fb34f3be9578452136e into 719ea14418834b3e92db927fc72dea9c(size=31.2 K), total size for store is 78.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:09,564 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:09,564 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/A, priority=13, startTime=1733193248630; duration=0sec 2024-12-03T02:34:09,564 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:09,564 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:A 2024-12-03T02:34:09,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T02:34:09,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:09,624 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T02:34:09,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:09,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:09,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:09,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:09,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:09,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:09,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d5b9861e1aa3464bb1ec9fa156885e7c_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193248728/Put/seqid=0 2024-12-03T02:34:09,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742020_1196 (size=12454) 2024-12-03T02:34:09,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:09,735 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d5b9861e1aa3464bb1ec9fa156885e7c_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d5b9861e1aa3464bb1ec9fa156885e7c_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:09,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/21852b8309624807a45bb9c29a82591b, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:09,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/21852b8309624807a45bb9c29a82591b is 175, key is test_row_0/A:col10/1733193248728/Put/seqid=0 2024-12-03T02:34:09,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742021_1197 (size=31255) 2024-12-03T02:34:09,775 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/21852b8309624807a45bb9c29a82591b 2024-12-03T02:34:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T02:34:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/73baff7aa179408b86a1e263c3fdf7d4 is 50, key is test_row_0/B:col10/1733193248728/Put/seqid=0 2024-12-03T02:34:09,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:09,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742022_1198 (size=12301) 2024-12-03T02:34:09,874 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/73baff7aa179408b86a1e263c3fdf7d4 2024-12-03T02:34:09,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/6787460f070b4da088e7da143a0dc549 is 50, key is test_row_0/C:col10/1733193248728/Put/seqid=0 2024-12-03T02:34:09,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742023_1199 (size=12301) 2024-12-03T02:34:09,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193249909, max timestamp=1733193249925Num uniques 859; 2024-12-03T02:34:09,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:09,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193249909, max timestamp=1733193249925Num uniques 859; 2024-12-03T02:34:09,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:09,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193249909, max timestamp=1733193249925Num uniques 859; 2024-12-03T02:34:09,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:09,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193309969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:09,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193309974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:09,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:10,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193310085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193310085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:10,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193310294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193310294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:10,314 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/6787460f070b4da088e7da143a0dc549 2024-12-03T02:34:10,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/21852b8309624807a45bb9c29a82591b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/21852b8309624807a45bb9c29a82591b 2024-12-03T02:34:10,345 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/21852b8309624807a45bb9c29a82591b, entries=150, sequenceid=360, filesize=30.5 K 2024-12-03T02:34:10,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/73baff7aa179408b86a1e263c3fdf7d4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/73baff7aa179408b86a1e263c3fdf7d4 2024-12-03T02:34:10,351 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/73baff7aa179408b86a1e263c3fdf7d4, entries=150, sequenceid=360, filesize=12.0 K 2024-12-03T02:34:10,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/6787460f070b4da088e7da143a0dc549 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/6787460f070b4da088e7da143a0dc549 2024-12-03T02:34:10,359 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/6787460f070b4da088e7da143a0dc549, entries=150, sequenceid=360, filesize=12.0 K 2024-12-03T02:34:10,360 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 64446d8651566fb34f3be9578452136e in 737ms, sequenceid=360, compaction requested=true 2024-12-03T02:34:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-03T02:34:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-03T02:34:10,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-03T02:34:10,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7020 sec 2024-12-03T02:34:10,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7090 sec 2024-12-03T02:34:10,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193249963, max timestamp=1733193250608Num uniques 1118; 2024-12-03T02:34:10,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193249963, max timestamp=1733193250608Num uniques 1118; 2024-12-03T02:34:10,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T02:34:10,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T02:34:10,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193249963, max timestamp=1733193250608Num uniques 1118; 2024-12-03T02:34:10,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T02:34:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:10,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-03T02:34:10,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:10,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:10,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:10,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:10,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:10,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:10,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e7c9295e8e63497d9542053b29d41cfa_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193250612/Put/seqid=0 2024-12-03T02:34:10,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742024_1200 (size=12454) 2024-12-03T02:34:10,678 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:10,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e7c9295e8e63497d9542053b29d41cfa_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e7c9295e8e63497d9542053b29d41cfa_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:10,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/bb307e20be96465da41c121345c849d7, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:10,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/bb307e20be96465da41c121345c849d7 is 175, key is test_row_0/A:col10/1733193250612/Put/seqid=0 2024-12-03T02:34:10,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742025_1201 (size=31255) 2024-12-03T02:34:10,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193250661, max timestamp=1733193250708Num uniques 859; 2024-12-03T02:34:10,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:10,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193250661, max timestamp=1733193250708Num uniques 859; 2024-12-03T02:34:10,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:10,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193250661, max timestamp=1733193250708Num uniques 859; 2024-12-03T02:34:10,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:10,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193310777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:10,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193310783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T02:34:10,816 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:10,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-03T02:34:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T02:34:10,823 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:10,824 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:10,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:10,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193310894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:10,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193310895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T02:34:10,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T02:34:10,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:10,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:10,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:10,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:10,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,108 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/bb307e20be96465da41c121345c849d7 2024-12-03T02:34:11,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193311104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193311104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:11,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/aebf5e4066104301a8e893f6d9e58df1 is 50, key is test_row_0/B:col10/1733193250612/Put/seqid=0 2024-12-03T02:34:11,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742026_1202 (size=12301) 2024-12-03T02:34:11,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T02:34:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T02:34:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T02:34:11,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:11,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193311414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:11,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193311414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:11,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T02:34:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T02:34:11,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/aebf5e4066104301a8e893f6d9e58df1 2024-12-03T02:34:11,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/3756d3f4c296475f92982be2b0fb296f is 50, key is test_row_0/C:col10/1733193250612/Put/seqid=0 2024-12-03T02:34:11,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742027_1203 (size=12301) 2024-12-03T02:34:11,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T02:34:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T02:34:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T02:34:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:11,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:11,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193311926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:11,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193311936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:11,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:11,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/3756d3f4c296475f92982be2b0fb296f 2024-12-03T02:34:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T02:34:11,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/bb307e20be96465da41c121345c849d7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/bb307e20be96465da41c121345c849d7 2024-12-03T02:34:11,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/bb307e20be96465da41c121345c849d7, entries=150, sequenceid=402, filesize=30.5 K 2024-12-03T02:34:11,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/aebf5e4066104301a8e893f6d9e58df1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/aebf5e4066104301a8e893f6d9e58df1 2024-12-03T02:34:12,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/aebf5e4066104301a8e893f6d9e58df1, entries=150, sequenceid=402, filesize=12.0 K 2024-12-03T02:34:12,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/3756d3f4c296475f92982be2b0fb296f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/3756d3f4c296475f92982be2b0fb296f 2024-12-03T02:34:12,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T02:34:12,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:12,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:12,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:12,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:12,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:12,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/3756d3f4c296475f92982be2b0fb296f, entries=150, sequenceid=402, filesize=12.0 K 2024-12-03T02:34:12,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=140.89 KB/144270 for 64446d8651566fb34f3be9578452136e in 1462ms, sequenceid=402, compaction requested=true 2024-12-03T02:34:12,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:12,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:12,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:12,084 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:12,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:12,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:12,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:12,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:34:12,085 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:12,096 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143052 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:12,096 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/A is initiating minor compaction (all files) 2024-12-03T02:34:12,096 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/A in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:12,096 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/719ea14418834b3e92db927fc72dea9c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d3d9cd166a244907a9efb02451699753, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/21852b8309624807a45bb9c29a82591b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/bb307e20be96465da41c121345c849d7] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=139.7 K 2024-12-03T02:34:12,096 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:12,096 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/719ea14418834b3e92db927fc72dea9c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d3d9cd166a244907a9efb02451699753, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/21852b8309624807a45bb9c29a82591b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/bb307e20be96465da41c121345c849d7] 2024-12-03T02:34:12,099 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:12,099 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/B is initiating minor compaction (all files) 2024-12-03T02:34:12,099 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/B in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:12,100 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/69ae1f3e0d494d20b4de2935d347dc4f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/ffb4739e76884fb89ee81a7b2af78420, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/73baff7aa179408b86a1e263c3fdf7d4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/aebf5e4066104301a8e893f6d9e58df1] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=48.7 K 2024-12-03T02:34:12,103 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69ae1f3e0d494d20b4de2935d347dc4f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733193248375 2024-12-03T02:34:12,104 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffb4739e76884fb89ee81a7b2af78420, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733193248574 2024-12-03T02:34:12,104 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73baff7aa179408b86a1e263c3fdf7d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733193248722 2024-12-03T02:34:12,105 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting aebf5e4066104301a8e893f6d9e58df1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733193250612 2024-12-03T02:34:12,107 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 719ea14418834b3e92db927fc72dea9c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733193248375 2024-12-03T02:34:12,119 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d9cd166a244907a9efb02451699753, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733193248563 2024-12-03T02:34:12,123 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 21852b8309624807a45bb9c29a82591b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733193248722 2024-12-03T02:34:12,128 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting bb307e20be96465da41c121345c849d7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733193250612 2024-12-03T02:34:12,133 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:12,133 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/3b6597a6a21c49ffabc93dcb2a3e6efb is 50, key is test_row_0/B:col10/1733193250612/Put/seqid=0 2024-12-03T02:34:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742028_1204 (size=13085) 2024-12-03T02:34:12,160 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:12,167 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203945668c5a11e46bf8b28bc4737430b76_64446d8651566fb34f3be9578452136e store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:12,174 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203945668c5a11e46bf8b28bc4737430b76_64446d8651566fb34f3be9578452136e, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:12,174 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203945668c5a11e46bf8b28bc4737430b76_64446d8651566fb34f3be9578452136e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:12,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742029_1205 (size=4469) 2024-12-03T02:34:12,195 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#A#compaction#166 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:12,196 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ffaf3f58652644a8a1abaccd3577323d is 175, key is test_row_0/A:col10/1733193250612/Put/seqid=0 2024-12-03T02:34:12,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742030_1206 (size=32039) 2024-12-03T02:34:12,213 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ffaf3f58652644a8a1abaccd3577323d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffaf3f58652644a8a1abaccd3577323d 2024-12-03T02:34:12,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T02:34:12,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:12,221 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T02:34:12,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:12,224 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64446d8651566fb34f3be9578452136e/A of 64446d8651566fb34f3be9578452136e into ffaf3f58652644a8a1abaccd3577323d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:12,225 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:12,225 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/A, priority=12, startTime=1733193252083; duration=0sec 2024-12-03T02:34:12,225 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:12,226 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:A 2024-12-03T02:34:12,226 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:12,229 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:12,229 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/C is initiating minor compaction (all files) 2024-12-03T02:34:12,229 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/C in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:12,229 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/87f76f5a45054ae9a7e7c118aa6854b1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/54c1f4886ba543ee9a0ee6949c923a94, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/6787460f070b4da088e7da143a0dc549, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/3756d3f4c296475f92982be2b0fb296f] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=48.7 K 2024-12-03T02:34:12,229 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f76f5a45054ae9a7e7c118aa6854b1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1733193248375 2024-12-03T02:34:12,230 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c1f4886ba543ee9a0ee6949c923a94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733193248574 2024-12-03T02:34:12,230 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 6787460f070b4da088e7da143a0dc549, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733193248722 2024-12-03T02:34:12,230 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 3756d3f4c296475f92982be2b0fb296f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733193250612 2024-12-03T02:34:12,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120358a10f37f7b24c4b8712238dedd33e31_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193250758/Put/seqid=0 2024-12-03T02:34:12,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742031_1207 (size=12454) 2024-12-03T02:34:12,241 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:12,242 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/9ae86728ae2042199512858023f652d2 is 50, key is test_row_0/C:col10/1733193250612/Put/seqid=0 2024-12-03T02:34:12,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742032_1208 (size=13085) 2024-12-03T02:34:12,563 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/3b6597a6a21c49ffabc93dcb2a3e6efb as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/3b6597a6a21c49ffabc93dcb2a3e6efb 2024-12-03T02:34:12,570 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64446d8651566fb34f3be9578452136e/B of 64446d8651566fb34f3be9578452136e into 3b6597a6a21c49ffabc93dcb2a3e6efb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:12,570 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:12,570 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/B, priority=12, startTime=1733193252084; duration=0sec 2024-12-03T02:34:12,570 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:12,570 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:B 2024-12-03T02:34:12,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:12,646 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120358a10f37f7b24c4b8712238dedd33e31_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120358a10f37f7b24c4b8712238dedd33e31_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:12,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ffb2626260fa443d806a7b974f91132f, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:12,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ffb2626260fa443d806a7b974f91132f is 175, key is test_row_0/A:col10/1733193250758/Put/seqid=0 2024-12-03T02:34:12,654 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/9ae86728ae2042199512858023f652d2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/9ae86728ae2042199512858023f652d2 2024-12-03T02:34:12,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742033_1209 (size=31255) 2024-12-03T02:34:12,657 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ffb2626260fa443d806a7b974f91132f 2024-12-03T02:34:12,663 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 64446d8651566fb34f3be9578452136e/C of 64446d8651566fb34f3be9578452136e into 9ae86728ae2042199512858023f652d2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:12,663 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:12,663 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/C, priority=12, startTime=1733193252084; duration=0sec 2024-12-03T02:34:12,663 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:12,663 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:C 2024-12-03T02:34:12,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/102c79eda34f4a529d87f9ca54bffa1c is 50, key is test_row_0/B:col10/1733193250758/Put/seqid=0 2024-12-03T02:34:12,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742034_1210 (size=12301) 2024-12-03T02:34:12,671 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/102c79eda34f4a529d87f9ca54bffa1c 2024-12-03T02:34:12,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/bd3a2c8ac2f2465187b423b58da61fdd is 50, key is test_row_0/C:col10/1733193250758/Put/seqid=0 2024-12-03T02:34:12,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742035_1211 (size=12301) 2024-12-03T02:34:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:12,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. as already flushing 2024-12-03T02:34:12,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:12,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193312948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:12,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:12,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193312949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T02:34:13,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193313055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193313065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,090 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/bd3a2c8ac2f2465187b423b58da61fdd 2024-12-03T02:34:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ffb2626260fa443d806a7b974f91132f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffb2626260fa443d806a7b974f91132f 2024-12-03T02:34:13,104 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffb2626260fa443d806a7b974f91132f, entries=150, sequenceid=439, filesize=30.5 K 2024-12-03T02:34:13,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/102c79eda34f4a529d87f9ca54bffa1c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/102c79eda34f4a529d87f9ca54bffa1c 2024-12-03T02:34:13,110 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/102c79eda34f4a529d87f9ca54bffa1c, entries=150, sequenceid=439, filesize=12.0 K 2024-12-03T02:34:13,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/bd3a2c8ac2f2465187b423b58da61fdd as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bd3a2c8ac2f2465187b423b58da61fdd 2024-12-03T02:34:13,117 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bd3a2c8ac2f2465187b423b58da61fdd, entries=150, sequenceid=439, filesize=12.0 K 2024-12-03T02:34:13,118 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 64446d8651566fb34f3be9578452136e in 897ms, sequenceid=439, compaction requested=false 2024-12-03T02:34:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-03T02:34:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-03T02:34:13,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-03T02:34:13,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2950 sec 2024-12-03T02:34:13,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3070 sec 2024-12-03T02:34:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:13,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:34:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:13,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:13,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:13,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:13,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:13,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034d7cc6c685424a6cbd541708c8ed9d9a_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193252943/Put/seqid=0 2024-12-03T02:34:13,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742036_1212 (size=14994) 2024-12-03T02:34:13,192 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:13,199 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034d7cc6c685424a6cbd541708c8ed9d9a_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034d7cc6c685424a6cbd541708c8ed9d9a_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:13,200 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/665b0e6b9c1b42eaadf9183350128b14, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:13,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/665b0e6b9c1b42eaadf9183350128b14 is 175, key is test_row_0/A:col10/1733193252943/Put/seqid=0 2024-12-03T02:34:13,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742037_1213 (size=39949) 2024-12-03T02:34:13,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193253202, max timestamp=1733193253224Num uniques 859; 2024-12-03T02:34:13,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:13,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193253202, max timestamp=1733193253224Num uniques 859; 2024-12-03T02:34:13,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193253202, max timestamp=1733193253224Num uniques 859; 2024-12-03T02:34:13,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:13,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:34:13,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193313262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193313266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193313271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193313271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193313275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193313375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193313388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193313388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193313575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193313584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193313584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193313605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193313605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,611 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=452, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/665b0e6b9c1b42eaadf9183350128b14 2024-12-03T02:34:13,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/399556104f99431faa27b03a42337eaf is 50, key is test_row_0/B:col10/1733193252943/Put/seqid=0 2024-12-03T02:34:13,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742038_1214 (size=12301) 2024-12-03T02:34:13,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193313894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193313915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193313915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/399556104f99431faa27b03a42337eaf 2024-12-03T02:34:14,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/2883171447184b31afd45d19e8499e49 is 50, key is test_row_0/C:col10/1733193252943/Put/seqid=0 2024-12-03T02:34:14,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742039_1215 (size=12301) 2024-12-03T02:34:14,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/2883171447184b31afd45d19e8499e49 2024-12-03T02:34:14,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/665b0e6b9c1b42eaadf9183350128b14 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/665b0e6b9c1b42eaadf9183350128b14 2024-12-03T02:34:14,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/665b0e6b9c1b42eaadf9183350128b14, entries=200, sequenceid=452, filesize=39.0 K 2024-12-03T02:34:14,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/399556104f99431faa27b03a42337eaf as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/399556104f99431faa27b03a42337eaf 2024-12-03T02:34:14,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/399556104f99431faa27b03a42337eaf, entries=150, sequenceid=452, filesize=12.0 K 2024-12-03T02:34:14,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/2883171447184b31afd45d19e8499e49 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/2883171447184b31afd45d19e8499e49 2024-12-03T02:34:14,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/2883171447184b31afd45d19e8499e49, entries=150, sequenceid=452, filesize=12.0 K 2024-12-03T02:34:14,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 64446d8651566fb34f3be9578452136e in 913ms, sequenceid=452, compaction requested=true 2024-12-03T02:34:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:14,079 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:14,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:14,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:14,080 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:14,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:14,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:14,081 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:14,081 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/B is initiating minor compaction (all files) 2024-12-03T02:34:14,081 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/B in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:14,081 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/3b6597a6a21c49ffabc93dcb2a3e6efb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/102c79eda34f4a529d87f9ca54bffa1c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/399556104f99431faa27b03a42337eaf] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=36.8 K 2024-12-03T02:34:14,081 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:14,081 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/A is initiating minor compaction (all files) 2024-12-03T02:34:14,081 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/A in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:14,081 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffaf3f58652644a8a1abaccd3577323d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffb2626260fa443d806a7b974f91132f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/665b0e6b9c1b42eaadf9183350128b14] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=100.8 K 2024-12-03T02:34:14,081 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:14,081 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffaf3f58652644a8a1abaccd3577323d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffb2626260fa443d806a7b974f91132f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/665b0e6b9c1b42eaadf9183350128b14] 2024-12-03T02:34:14,082 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffaf3f58652644a8a1abaccd3577323d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733193250612 2024-12-03T02:34:14,082 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b6597a6a21c49ffabc93dcb2a3e6efb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733193250612 2024-12-03T02:34:14,083 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 102c79eda34f4a529d87f9ca54bffa1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733193250758 2024-12-03T02:34:14,083 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffb2626260fa443d806a7b974f91132f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733193250758 2024-12-03T02:34:14,083 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 399556104f99431faa27b03a42337eaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733193252943 2024-12-03T02:34:14,083 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 665b0e6b9c1b42eaadf9183350128b14, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733193252943 2024-12-03T02:34:14,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-03T02:34:14,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:14,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:14,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:14,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:14,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:14,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:14,092 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#B#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:14,093 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/78eae6357c1942c8a8c440a544f65e51 is 50, key is test_row_0/B:col10/1733193252943/Put/seqid=0 2024-12-03T02:34:14,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193314112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193314117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,124 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:14,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203daa4ad15e9ca41cfbc1603aebf5c14a4_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193253255/Put/seqid=0 2024-12-03T02:34:14,132 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c25a4a03c1674ac09e4e65c3405b08ce_64446d8651566fb34f3be9578452136e store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:14,134 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c25a4a03c1674ac09e4e65c3405b08ce_64446d8651566fb34f3be9578452136e, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:14,134 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c25a4a03c1674ac09e4e65c3405b08ce_64446d8651566fb34f3be9578452136e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:14,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742041_1217 (size=12454) 2024-12-03T02:34:14,168 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:14,177 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203daa4ad15e9ca41cfbc1603aebf5c14a4_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203daa4ad15e9ca41cfbc1603aebf5c14a4_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:14,178 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/35977bbbb5764af8b839b4a3a8ccce37, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:14,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742040_1216 (size=13187) 2024-12-03T02:34:14,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/35977bbbb5764af8b839b4a3a8ccce37 is 175, key is test_row_0/A:col10/1733193253255/Put/seqid=0 2024-12-03T02:34:14,190 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/78eae6357c1942c8a8c440a544f65e51 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/78eae6357c1942c8a8c440a544f65e51 2024-12-03T02:34:14,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742042_1218 (size=4469) 2024-12-03T02:34:14,198 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#A#compaction#175 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:14,199 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/343ba7be5dcb49cf8c6cda997454f794 is 175, key is test_row_0/A:col10/1733193252943/Put/seqid=0 2024-12-03T02:34:14,201 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/B of 64446d8651566fb34f3be9578452136e into 78eae6357c1942c8a8c440a544f65e51(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:14,201 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:14,201 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/B, priority=13, startTime=1733193254079; duration=0sec 2024-12-03T02:34:14,201 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:14,201 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:B 2024-12-03T02:34:14,201 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:14,202 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:14,203 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 64446d8651566fb34f3be9578452136e/C is initiating minor compaction (all files) 2024-12-03T02:34:14,203 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64446d8651566fb34f3be9578452136e/C in TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:14,203 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/9ae86728ae2042199512858023f652d2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bd3a2c8ac2f2465187b423b58da61fdd, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/2883171447184b31afd45d19e8499e49] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp, totalSize=36.8 K 2024-12-03T02:34:14,203 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae86728ae2042199512858023f652d2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733193250612 2024-12-03T02:34:14,204 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3a2c8ac2f2465187b423b58da61fdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733193250758 2024-12-03T02:34:14,204 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2883171447184b31afd45d19e8499e49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1733193252943 2024-12-03T02:34:14,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742043_1219 (size=31255) 2024-12-03T02:34:14,225 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64446d8651566fb34f3be9578452136e#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:14,225 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/173263ecbef048a4851693152b4a263a is 50, key is test_row_0/C:col10/1733193252943/Put/seqid=0 2024-12-03T02:34:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193314225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193314226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742044_1220 (size=32141) 2024-12-03T02:34:14,248 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/343ba7be5dcb49cf8c6cda997454f794 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/343ba7be5dcb49cf8c6cda997454f794 2024-12-03T02:34:14,255 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/A of 64446d8651566fb34f3be9578452136e into 343ba7be5dcb49cf8c6cda997454f794(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:14,255 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:14,255 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/A, priority=13, startTime=1733193254079; duration=0sec 2024-12-03T02:34:14,255 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:14,255 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:A 2024-12-03T02:34:14,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742045_1221 (size=13187) 2024-12-03T02:34:14,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193314415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193314426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193314426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193314435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193314435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,623 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=489, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/35977bbbb5764af8b839b4a3a8ccce37 2024-12-03T02:34:14,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/cfa2a0701a564225b13f15c9362a797f is 50, key is test_row_0/B:col10/1733193253255/Put/seqid=0 2024-12-03T02:34:14,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742046_1222 (size=12301) 2024-12-03T02:34:14,675 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/173263ecbef048a4851693152b4a263a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/173263ecbef048a4851693152b4a263a 2024-12-03T02:34:14,679 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64446d8651566fb34f3be9578452136e/C of 64446d8651566fb34f3be9578452136e into 173263ecbef048a4851693152b4a263a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:14,679 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:14,679 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., storeName=64446d8651566fb34f3be9578452136e/C, priority=13, startTime=1733193254080; duration=0sec 2024-12-03T02:34:14,680 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:14,680 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:C 2024-12-03T02:34:14,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193314744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,746 INFO [Thread-584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-584. 2024-12-03T02:34:14,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:14,746 DEBUG [Thread-584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,746 DEBUG [Thread-584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193314745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,746 DEBUG [Thread-584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:14,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:14,747 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-12-03T02:34:14,747 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:14,747 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:14,747 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:14,747 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:14,754 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-12-03T02:34:14,754 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:14,754 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:14,754 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:14,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:14,757 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-12-03T02:34:14,757 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:14,757 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:14,757 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:14,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T02:34:14,984 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:15,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/cfa2a0701a564225b13f15c9362a797f 2024-12-03T02:34:15,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/1d222a5af75c47cfaf381957bbc7bc31 is 50, key is test_row_0/C:col10/1733193253255/Put/seqid=0 2024-12-03T02:34:15,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742047_1223 (size=12301) 2024-12-03T02:34:15,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34202 deadline: 1733193315255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:15,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34208 deadline: 1733193315256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:15,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34210 deadline: 1733193315424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:15,425 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:15,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:15,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34192 deadline: 1733193315445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34150 deadline: 1733193315445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:15,446 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:15,446 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in 64446d8651566fb34f3be9578452136e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:15,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/1d222a5af75c47cfaf381957bbc7bc31 2024-12-03T02:34:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/35977bbbb5764af8b839b4a3a8ccce37 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/35977bbbb5764af8b839b4a3a8ccce37 2024-12-03T02:34:15,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/35977bbbb5764af8b839b4a3a8ccce37, entries=150, sequenceid=489, filesize=30.5 K 2024-12-03T02:34:15,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/cfa2a0701a564225b13f15c9362a797f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/cfa2a0701a564225b13f15c9362a797f 2024-12-03T02:34:15,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/cfa2a0701a564225b13f15c9362a797f, entries=150, sequenceid=489, filesize=12.0 K 2024-12-03T02:34:15,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/1d222a5af75c47cfaf381957bbc7bc31 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d222a5af75c47cfaf381957bbc7bc31 2024-12-03T02:34:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d222a5af75c47cfaf381957bbc7bc31, entries=150, sequenceid=489, filesize=12.0 K 2024-12-03T02:34:15,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 64446d8651566fb34f3be9578452136e in 1388ms, sequenceid=489, compaction requested=false 2024-12-03T02:34:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:16,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:34:16,267 INFO [Thread-579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-579. 2024-12-03T02:34:16,267 INFO [Thread-575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-575. 2024-12-03T02:34:16,267 DEBUG [Thread-579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:16,267 DEBUG [Thread-575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:16,267 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:16,267 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:16,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:16,267 DEBUG [Thread-575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:16,267 DEBUG [Thread-579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:16,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:16,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:16,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:16,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:16,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:16,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:16,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:16,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c9f5f6e465ee4fe88f7d0e7ff1e79a56_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193254114/Put/seqid=0 2024-12-03T02:34:16,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742048_1224 (size=12454) 2024-12-03T02:34:16,678 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:16,683 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c9f5f6e465ee4fe88f7d0e7ff1e79a56_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c9f5f6e465ee4fe88f7d0e7ff1e79a56_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:16,684 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ad3a9f59d1934e1fbf201a684f997314, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:16,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ad3a9f59d1934e1fbf201a684f997314 is 175, key is test_row_0/A:col10/1733193254114/Put/seqid=0 2024-12-03T02:34:16,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742049_1225 (size=31255) 2024-12-03T02:34:17,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=502, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ad3a9f59d1934e1fbf201a684f997314 2024-12-03T02:34:17,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/8a74f3ad182d42baab0d28f27698305c is 50, key is test_row_0/B:col10/1733193254114/Put/seqid=0 2024-12-03T02:34:17,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742050_1226 (size=12301) 2024-12-03T02:34:17,437 INFO [Thread-573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-573. 2024-12-03T02:34:17,437 DEBUG [Thread-573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:17,437 DEBUG [Thread-573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:17,437 DEBUG [Thread-573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:17,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:17,456 INFO [Thread-581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-581. 2024-12-03T02:34:17,456 DEBUG [Thread-581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:17,456 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:17,456 DEBUG [Thread-581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:17,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:17,477 INFO [Thread-577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-577. 2024-12-03T02:34:17,477 DEBUG [Thread-577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:17,477 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:17,477 DEBUG [Thread-577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:17,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 167 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 188 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3152 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3048 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1400 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4191 rows 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1391 2024-12-03T02:34:17,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4158 rows 2024-12-03T02:34:17,478 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:34:17,478 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:34:17,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:17,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:17,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:17,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T02:34:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T02:34:17,485 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193257485"}]},"ts":"1733193257485"} 2024-12-03T02:34:17,487 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T02:34:17,487 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T02:34:17,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T02:34:17,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, UNASSIGN}] 2024-12-03T02:34:17,491 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, UNASSIGN 2024-12-03T02:34:17,492 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=64446d8651566fb34f3be9578452136e, regionState=CLOSING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:17,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, UNASSIGN because future has completed 2024-12-03T02:34:17,494 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:34:17,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:34:17,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/8a74f3ad182d42baab0d28f27698305c 2024-12-03T02:34:17,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/acb960c36a8b44bd9637115aa335eea4 is 50, key is test_row_0/C:col10/1733193254114/Put/seqid=0 2024-12-03T02:34:17,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742051_1227 (size=12301) 2024-12-03T02:34:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T02:34:17,647 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:17,647 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:34:17,647 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 64446d8651566fb34f3be9578452136e, disabling compactions & flushes 2024-12-03T02:34:17,647 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T02:34:17,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/acb960c36a8b44bd9637115aa335eea4 2024-12-03T02:34:17,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/ad3a9f59d1934e1fbf201a684f997314 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ad3a9f59d1934e1fbf201a684f997314 2024-12-03T02:34:17,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ad3a9f59d1934e1fbf201a684f997314, entries=150, sequenceid=502, filesize=30.5 K 2024-12-03T02:34:17,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/8a74f3ad182d42baab0d28f27698305c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/8a74f3ad182d42baab0d28f27698305c 2024-12-03T02:34:17,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/8a74f3ad182d42baab0d28f27698305c, entries=150, sequenceid=502, filesize=12.0 K 2024-12-03T02:34:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/acb960c36a8b44bd9637115aa335eea4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/acb960c36a8b44bd9637115aa335eea4 2024-12-03T02:34:17,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/acb960c36a8b44bd9637115aa335eea4, entries=150, sequenceid=502, filesize=12.0 K 2024-12-03T02:34:17,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 64446d8651566fb34f3be9578452136e in 1699ms, sequenceid=502, compaction requested=true 2024-12-03T02:34:17,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 64446d8651566fb34f3be9578452136e: 2024-12-03T02:34:17,966 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:17,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:17,966 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:17,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:17,966 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. because compaction request was cancelled 2024-12-03T02:34:17,966 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. after waiting 0 ms 2024-12-03T02:34:17,966 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:17,966 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:A 2024-12-03T02:34:17,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:17,966 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. because compaction request was cancelled 2024-12-03T02:34:17,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:17,966 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:B 2024-12-03T02:34:17,966 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 64446d8651566fb34f3be9578452136e 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-03T02:34:17,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 64446d8651566fb34f3be9578452136e:C, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:17,966 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. because compaction request was cancelled 2024-12-03T02:34:17,966 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64446d8651566fb34f3be9578452136e:C 2024-12-03T02:34:17,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:17,966 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=A 2024-12-03T02:34:17,967 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:17,967 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=B 2024-12-03T02:34:17,967 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:17,967 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 64446d8651566fb34f3be9578452136e, store=C 2024-12-03T02:34:17,967 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:17,972 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120382836340c6a04c108ef9bc8cc2d71987_64446d8651566fb34f3be9578452136e is 50, key is test_row_0/A:col10/1733193257455/Put/seqid=0 2024-12-03T02:34:17,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742052_1228 (size=9914) 2024-12-03T02:34:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T02:34:18,376 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:18,382 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120382836340c6a04c108ef9bc8cc2d71987_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120382836340c6a04c108ef9bc8cc2d71987_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:18,383 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/d4c7a5c29bd94690bb737decdddd798e, store: [table=TestAcidGuarantees family=A region=64446d8651566fb34f3be9578452136e] 2024-12-03T02:34:18,384 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/d4c7a5c29bd94690bb737decdddd798e is 175, key is test_row_0/A:col10/1733193257455/Put/seqid=0 2024-12-03T02:34:18,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742053_1229 (size=22561) 2024-12-03T02:34:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T02:34:18,788 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=508, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/d4c7a5c29bd94690bb737decdddd798e 2024-12-03T02:34:18,798 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/cb170cec1afc420fa398903cb4f7cd04 is 50, key is test_row_0/B:col10/1733193257455/Put/seqid=0 2024-12-03T02:34:18,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742054_1230 (size=9857) 2024-12-03T02:34:19,205 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/cb170cec1afc420fa398903cb4f7cd04 2024-12-03T02:34:19,212 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/b64fb662e1af4a70a4e319e10ed11bf6 is 50, key is test_row_0/C:col10/1733193257455/Put/seqid=0 2024-12-03T02:34:19,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742055_1231 (size=9857) 2024-12-03T02:34:19,616 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/b64fb662e1af4a70a4e319e10ed11bf6 2024-12-03T02:34:19,620 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/A/d4c7a5c29bd94690bb737decdddd798e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d4c7a5c29bd94690bb737decdddd798e 2024-12-03T02:34:19,624 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d4c7a5c29bd94690bb737decdddd798e, entries=100, sequenceid=508, filesize=22.0 K 2024-12-03T02:34:19,625 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/B/cb170cec1afc420fa398903cb4f7cd04 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/cb170cec1afc420fa398903cb4f7cd04 2024-12-03T02:34:19,629 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/cb170cec1afc420fa398903cb4f7cd04, entries=100, sequenceid=508, filesize=9.6 K 2024-12-03T02:34:19,629 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/.tmp/C/b64fb662e1af4a70a4e319e10ed11bf6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/b64fb662e1af4a70a4e319e10ed11bf6 2024-12-03T02:34:19,633 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/b64fb662e1af4a70a4e319e10ed11bf6, entries=100, sequenceid=508, filesize=9.6 K 2024-12-03T02:34:19,634 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 64446d8651566fb34f3be9578452136e in 1668ms, sequenceid=508, compaction requested=true 2024-12-03T02:34:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T02:34:19,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb91356c5e6e4490afd493dc4201feb2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/191e34ed0c1b4e15897f9ced4740add1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/920b21e3e1264cf1bf2f96626b55ac8a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7d7d283607894ea2bca81790ce4d0283, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/fbe4a3024087488bae9ad17a656be48f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb6bc91600204400b5221afe46976ad6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5734bd349f2d4057908dd3c20a70d3e5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7f45ccd57f3946f7895e07c5aa5a45dc, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/adbe6fc00ce049c5aadf7ebb753a245f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/75876d2b9f9647a1a6212f65e9286696, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/e5ae99affd3a4bc7809bfa9f2499a154, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/daea7e744c64424dadeacdf6d8fd9eb0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5ce16ddd849c419696911e34841e947d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/27940a1da0ac474ea56b635d7fa28dfe, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/727746d6cee34fc1b0050f18e73be44a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/c968b69e8fbe45bfbcfcf537c72d7d28, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/2683d48b35644077bb175e5c707731e8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/3fd7f7e94adc4a6d80b983c22e00c8f9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/a7678328bd80468d98d8a77e24aa5b22, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/719ea14418834b3e92db927fc72dea9c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d3d9cd166a244907a9efb02451699753, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/21852b8309624807a45bb9c29a82591b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffaf3f58652644a8a1abaccd3577323d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/bb307e20be96465da41c121345c849d7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffb2626260fa443d806a7b974f91132f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/665b0e6b9c1b42eaadf9183350128b14] to archive 2024-12-03T02:34:19,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:34:19,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb91356c5e6e4490afd493dc4201feb2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb91356c5e6e4490afd493dc4201feb2 2024-12-03T02:34:19,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/191e34ed0c1b4e15897f9ced4740add1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/191e34ed0c1b4e15897f9ced4740add1 2024-12-03T02:34:19,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/920b21e3e1264cf1bf2f96626b55ac8a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/920b21e3e1264cf1bf2f96626b55ac8a 2024-12-03T02:34:19,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7d7d283607894ea2bca81790ce4d0283 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7d7d283607894ea2bca81790ce4d0283 2024-12-03T02:34:19,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/fbe4a3024087488bae9ad17a656be48f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/fbe4a3024087488bae9ad17a656be48f 2024-12-03T02:34:19,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb6bc91600204400b5221afe46976ad6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/eb6bc91600204400b5221afe46976ad6 2024-12-03T02:34:19,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5734bd349f2d4057908dd3c20a70d3e5 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5734bd349f2d4057908dd3c20a70d3e5 2024-12-03T02:34:19,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7f45ccd57f3946f7895e07c5aa5a45dc to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/7f45ccd57f3946f7895e07c5aa5a45dc 2024-12-03T02:34:19,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/adbe6fc00ce049c5aadf7ebb753a245f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/adbe6fc00ce049c5aadf7ebb753a245f 2024-12-03T02:34:19,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/75876d2b9f9647a1a6212f65e9286696 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/75876d2b9f9647a1a6212f65e9286696 2024-12-03T02:34:19,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/e5ae99affd3a4bc7809bfa9f2499a154 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/e5ae99affd3a4bc7809bfa9f2499a154 2024-12-03T02:34:19,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/daea7e744c64424dadeacdf6d8fd9eb0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/daea7e744c64424dadeacdf6d8fd9eb0 2024-12-03T02:34:19,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5ce16ddd849c419696911e34841e947d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/5ce16ddd849c419696911e34841e947d 2024-12-03T02:34:19,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/27940a1da0ac474ea56b635d7fa28dfe to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/27940a1da0ac474ea56b635d7fa28dfe 2024-12-03T02:34:19,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/727746d6cee34fc1b0050f18e73be44a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/727746d6cee34fc1b0050f18e73be44a 2024-12-03T02:34:19,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/c968b69e8fbe45bfbcfcf537c72d7d28 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/c968b69e8fbe45bfbcfcf537c72d7d28 2024-12-03T02:34:19,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/2683d48b35644077bb175e5c707731e8 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/2683d48b35644077bb175e5c707731e8 2024-12-03T02:34:19,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/3fd7f7e94adc4a6d80b983c22e00c8f9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/3fd7f7e94adc4a6d80b983c22e00c8f9 2024-12-03T02:34:19,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/a7678328bd80468d98d8a77e24aa5b22 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/a7678328bd80468d98d8a77e24aa5b22 2024-12-03T02:34:19,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/719ea14418834b3e92db927fc72dea9c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/719ea14418834b3e92db927fc72dea9c 2024-12-03T02:34:19,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d3d9cd166a244907a9efb02451699753 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d3d9cd166a244907a9efb02451699753 2024-12-03T02:34:19,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/21852b8309624807a45bb9c29a82591b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/21852b8309624807a45bb9c29a82591b 2024-12-03T02:34:19,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffaf3f58652644a8a1abaccd3577323d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffaf3f58652644a8a1abaccd3577323d 2024-12-03T02:34:19,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/bb307e20be96465da41c121345c849d7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/bb307e20be96465da41c121345c849d7 2024-12-03T02:34:19,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffb2626260fa443d806a7b974f91132f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ffb2626260fa443d806a7b974f91132f 2024-12-03T02:34:19,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/665b0e6b9c1b42eaadf9183350128b14 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/665b0e6b9c1b42eaadf9183350128b14 2024-12-03T02:34:19,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/c95c1bea69f04167b7bad7ff4dec07f9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/7832afab0871420d99a59a049dd45773, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/52465c021d1946aa8478911ace9097cb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e869cdcb4d2149e1b04742f4cd343c2b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e935f79f20ce4f87a1ab42de3b17697d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5b2652c21d3f4607a6b48eb14b920006, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/047de2d5e90c4d4ca0642b03ebada8c9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d3b1b198dff34931b6948d506de08a3e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/33100ce00a194fc7ad00c992beedd343, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/2c0c8395773b45c38c4a2577dd3a578e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/09fa679c4e3d4b37b134ebb9045fb46a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/f8be8a9ad32548deae2bd091a4a9e501, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5204aed711054587a296cf765f0d7167, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/bcea4cd7fdc446e09b7cb9fad36f880e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/9b967a054ad44a5e894d466f4519de9f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/679e4790f1bd4b86a851b76cf303e413, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d77771546ed2429e811311355a589bff, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/4156aa05691647dc85ad7ba69f474394, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/69ae1f3e0d494d20b4de2935d347dc4f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/317492a851d0455596e6e2e12da7ca5c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/ffb4739e76884fb89ee81a7b2af78420, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/73baff7aa179408b86a1e263c3fdf7d4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/3b6597a6a21c49ffabc93dcb2a3e6efb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/aebf5e4066104301a8e893f6d9e58df1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/102c79eda34f4a529d87f9ca54bffa1c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/399556104f99431faa27b03a42337eaf] to archive 2024-12-03T02:34:19,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:34:19,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/c95c1bea69f04167b7bad7ff4dec07f9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/c95c1bea69f04167b7bad7ff4dec07f9 2024-12-03T02:34:19,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/7832afab0871420d99a59a049dd45773 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/7832afab0871420d99a59a049dd45773 2024-12-03T02:34:19,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/52465c021d1946aa8478911ace9097cb to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/52465c021d1946aa8478911ace9097cb 2024-12-03T02:34:19,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e869cdcb4d2149e1b04742f4cd343c2b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e869cdcb4d2149e1b04742f4cd343c2b 2024-12-03T02:34:19,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e935f79f20ce4f87a1ab42de3b17697d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/e935f79f20ce4f87a1ab42de3b17697d 2024-12-03T02:34:19,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5b2652c21d3f4607a6b48eb14b920006 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5b2652c21d3f4607a6b48eb14b920006 2024-12-03T02:34:19,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/047de2d5e90c4d4ca0642b03ebada8c9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/047de2d5e90c4d4ca0642b03ebada8c9 2024-12-03T02:34:19,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d3b1b198dff34931b6948d506de08a3e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d3b1b198dff34931b6948d506de08a3e 2024-12-03T02:34:19,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/33100ce00a194fc7ad00c992beedd343 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/33100ce00a194fc7ad00c992beedd343 2024-12-03T02:34:19,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/2c0c8395773b45c38c4a2577dd3a578e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/2c0c8395773b45c38c4a2577dd3a578e 2024-12-03T02:34:19,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/09fa679c4e3d4b37b134ebb9045fb46a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/09fa679c4e3d4b37b134ebb9045fb46a 2024-12-03T02:34:19,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/f8be8a9ad32548deae2bd091a4a9e501 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/f8be8a9ad32548deae2bd091a4a9e501 2024-12-03T02:34:19,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5204aed711054587a296cf765f0d7167 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/5204aed711054587a296cf765f0d7167 2024-12-03T02:34:19,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/bcea4cd7fdc446e09b7cb9fad36f880e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/bcea4cd7fdc446e09b7cb9fad36f880e 2024-12-03T02:34:19,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/9b967a054ad44a5e894d466f4519de9f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/9b967a054ad44a5e894d466f4519de9f 2024-12-03T02:34:19,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/679e4790f1bd4b86a851b76cf303e413 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/679e4790f1bd4b86a851b76cf303e413 2024-12-03T02:34:19,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d77771546ed2429e811311355a589bff to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/d77771546ed2429e811311355a589bff 2024-12-03T02:34:19,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/4156aa05691647dc85ad7ba69f474394 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/4156aa05691647dc85ad7ba69f474394 2024-12-03T02:34:19,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/69ae1f3e0d494d20b4de2935d347dc4f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/69ae1f3e0d494d20b4de2935d347dc4f 2024-12-03T02:34:19,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/317492a851d0455596e6e2e12da7ca5c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/317492a851d0455596e6e2e12da7ca5c 2024-12-03T02:34:19,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/ffb4739e76884fb89ee81a7b2af78420 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/ffb4739e76884fb89ee81a7b2af78420 2024-12-03T02:34:19,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/73baff7aa179408b86a1e263c3fdf7d4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/73baff7aa179408b86a1e263c3fdf7d4 2024-12-03T02:34:19,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/3b6597a6a21c49ffabc93dcb2a3e6efb to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/3b6597a6a21c49ffabc93dcb2a3e6efb 2024-12-03T02:34:19,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/aebf5e4066104301a8e893f6d9e58df1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/aebf5e4066104301a8e893f6d9e58df1 2024-12-03T02:34:19,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/102c79eda34f4a529d87f9ca54bffa1c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/102c79eda34f4a529d87f9ca54bffa1c 2024-12-03T02:34:19,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/399556104f99431faa27b03a42337eaf to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/399556104f99431faa27b03a42337eaf 2024-12-03T02:34:19,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d02acc8c10f45eab1a04e40a61cf71e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/41a75a4b03744c749a6996de6fe07f79, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/58c6024ad94147a2bb4ec4a96fbe3b3c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/d67ff727d118430c958b7ba0dffa9d96, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/19a0253660924708a37fa8e32055d2c1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/7ac40aeb8b8b45778453791b64bd3670, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a0d2bac3b1e94b8e9472c6446bba296e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/c42b0059f8d84d6c84f4898ea0cd4288, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/96526ebad643412eb9f8d34793f9bf8b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bec5a9a0bbd843d3ae7d5e2f5c3cc948, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/46af28506be24beebcf763a17f344ff6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/49df7ffc273748acbdf0c9b74babee45, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/08d22670842441079c78c52083081231, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/21537562ca2048cbb8d66a6cb5fc2249, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/db59487f536b4fb88152f6d94a898893, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/75939ccaea024ca9afe5f31f7591486d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a98ec502868f4f1395d41ab766740018, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/eaad8a75da9245b5ada9783c994e8fe7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/87f76f5a45054ae9a7e7c118aa6854b1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/f20b52ceafb64d8eabbbf8d2a16720fc, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/54c1f4886ba543ee9a0ee6949c923a94, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/6787460f070b4da088e7da143a0dc549, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/9ae86728ae2042199512858023f652d2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/3756d3f4c296475f92982be2b0fb296f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bd3a2c8ac2f2465187b423b58da61fdd, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/2883171447184b31afd45d19e8499e49] to archive 2024-12-03T02:34:19,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:34:19,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d02acc8c10f45eab1a04e40a61cf71e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d02acc8c10f45eab1a04e40a61cf71e 2024-12-03T02:34:19,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/41a75a4b03744c749a6996de6fe07f79 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/41a75a4b03744c749a6996de6fe07f79 2024-12-03T02:34:19,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/58c6024ad94147a2bb4ec4a96fbe3b3c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/58c6024ad94147a2bb4ec4a96fbe3b3c 2024-12-03T02:34:19,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/d67ff727d118430c958b7ba0dffa9d96 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/d67ff727d118430c958b7ba0dffa9d96 2024-12-03T02:34:19,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/19a0253660924708a37fa8e32055d2c1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/19a0253660924708a37fa8e32055d2c1 2024-12-03T02:34:19,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/7ac40aeb8b8b45778453791b64bd3670 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/7ac40aeb8b8b45778453791b64bd3670 2024-12-03T02:34:19,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a0d2bac3b1e94b8e9472c6446bba296e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a0d2bac3b1e94b8e9472c6446bba296e 2024-12-03T02:34:19,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/c42b0059f8d84d6c84f4898ea0cd4288 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/c42b0059f8d84d6c84f4898ea0cd4288 2024-12-03T02:34:19,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/96526ebad643412eb9f8d34793f9bf8b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/96526ebad643412eb9f8d34793f9bf8b 2024-12-03T02:34:19,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bec5a9a0bbd843d3ae7d5e2f5c3cc948 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bec5a9a0bbd843d3ae7d5e2f5c3cc948 2024-12-03T02:34:19,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/46af28506be24beebcf763a17f344ff6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/46af28506be24beebcf763a17f344ff6 2024-12-03T02:34:19,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/49df7ffc273748acbdf0c9b74babee45 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/49df7ffc273748acbdf0c9b74babee45 2024-12-03T02:34:19,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/08d22670842441079c78c52083081231 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/08d22670842441079c78c52083081231 2024-12-03T02:34:19,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/21537562ca2048cbb8d66a6cb5fc2249 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/21537562ca2048cbb8d66a6cb5fc2249 2024-12-03T02:34:19,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/db59487f536b4fb88152f6d94a898893 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/db59487f536b4fb88152f6d94a898893 2024-12-03T02:34:19,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/75939ccaea024ca9afe5f31f7591486d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/75939ccaea024ca9afe5f31f7591486d 2024-12-03T02:34:19,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a98ec502868f4f1395d41ab766740018 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/a98ec502868f4f1395d41ab766740018 2024-12-03T02:34:19,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/eaad8a75da9245b5ada9783c994e8fe7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/eaad8a75da9245b5ada9783c994e8fe7 2024-12-03T02:34:19,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/87f76f5a45054ae9a7e7c118aa6854b1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/87f76f5a45054ae9a7e7c118aa6854b1 2024-12-03T02:34:19,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/f20b52ceafb64d8eabbbf8d2a16720fc to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/f20b52ceafb64d8eabbbf8d2a16720fc 2024-12-03T02:34:19,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/54c1f4886ba543ee9a0ee6949c923a94 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/54c1f4886ba543ee9a0ee6949c923a94 2024-12-03T02:34:19,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/6787460f070b4da088e7da143a0dc549 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/6787460f070b4da088e7da143a0dc549 2024-12-03T02:34:19,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/9ae86728ae2042199512858023f652d2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/9ae86728ae2042199512858023f652d2 2024-12-03T02:34:19,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/3756d3f4c296475f92982be2b0fb296f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/3756d3f4c296475f92982be2b0fb296f 2024-12-03T02:34:19,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bd3a2c8ac2f2465187b423b58da61fdd to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/bd3a2c8ac2f2465187b423b58da61fdd 2024-12-03T02:34:19,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/2883171447184b31afd45d19e8499e49 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/2883171447184b31afd45d19e8499e49 2024-12-03T02:34:19,738 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/recovered.edits/511.seqid, newMaxSeqId=511, maxSeqId=4 2024-12-03T02:34:19,739 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. 2024-12-03T02:34:19,739 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 64446d8651566fb34f3be9578452136e: Waiting for close lock at 1733193257647Running coprocessor pre-close hooks at 1733193257647Disabling compacts and flushes for region at 1733193257647Disabling writes for close at 1733193257966 (+319 ms)Obtaining lock to block concurrent updates at 1733193257966Preparing flush snapshotting stores in 64446d8651566fb34f3be9578452136e at 1733193257966Finished memstore snapshotting TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733193257967 (+1 ms)Flushing stores of TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e. at 1733193257967Flushing 64446d8651566fb34f3be9578452136e/A: creating writer at 1733193257967Flushing mob file 64446d8651566fb34f3be9578452136e/A: appending metadata at 1733193257972 (+5 ms)Flushing mob file 64446d8651566fb34f3be9578452136e/A: closing flushed file at 1733193257972Flushing 64446d8651566fb34f3be9578452136e/A: appending metadata at 1733193258383 (+411 ms)Flushing 64446d8651566fb34f3be9578452136e/A: closing flushed file at 1733193258383Flushing 64446d8651566fb34f3be9578452136e/B: creating writer at 1733193258794 (+411 ms)Flushing 64446d8651566fb34f3be9578452136e/B: appending metadata at 1733193258798 (+4 ms)Flushing 64446d8651566fb34f3be9578452136e/B: closing flushed file at 1733193258798Flushing 64446d8651566fb34f3be9578452136e/C: creating writer at 1733193259209 (+411 ms)Flushing 64446d8651566fb34f3be9578452136e/C: appending metadata at 1733193259211 (+2 ms)Flushing 64446d8651566fb34f3be9578452136e/C: closing flushed file at 1733193259211Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c0d88fa: reopening flushed file at 1733193259619 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64cff7e5: reopening flushed file at 1733193259624 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1546a679: reopening flushed file at 1733193259629 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 64446d8651566fb34f3be9578452136e in 1668ms, sequenceid=508, compaction requested=true at 1733193259634 (+5 ms)Writing region close event to WAL at 1733193259732 (+98 ms)Running coprocessor post-close hooks at 1733193259739 (+7 ms)Closed at 1733193259739 2024-12-03T02:34:19,742 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 64446d8651566fb34f3be9578452136e 2024-12-03T02:34:19,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=64446d8651566fb34f3be9578452136e, regionState=CLOSED 2024-12-03T02:34:19,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:34:19,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-03T02:34:19,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 64446d8651566fb34f3be9578452136e, server=64035cbb0b46,38721,1733193202350 in 2.2540 sec 2024-12-03T02:34:19,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-12-03T02:34:19,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=64446d8651566fb34f3be9578452136e, UNASSIGN in 2.2600 sec 2024-12-03T02:34:19,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-03T02:34:19,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2650 sec 2024-12-03T02:34:19,755 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193259755"}]},"ts":"1733193259755"} 2024-12-03T02:34:19,757 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T02:34:19,757 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T02:34:19,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2770 sec 2024-12-03T02:34:20,003 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T02:34:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T02:34:21,655 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:21,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T02:34:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:21,658 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T02:34:21,659 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:21,663 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,668 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/recovered.edits] 2024-12-03T02:34:21,672 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/343ba7be5dcb49cf8c6cda997454f794 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/343ba7be5dcb49cf8c6cda997454f794 2024-12-03T02:34:21,674 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/35977bbbb5764af8b839b4a3a8ccce37 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/35977bbbb5764af8b839b4a3a8ccce37 2024-12-03T02:34:21,675 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ad3a9f59d1934e1fbf201a684f997314 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/ad3a9f59d1934e1fbf201a684f997314 2024-12-03T02:34:21,677 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d4c7a5c29bd94690bb737decdddd798e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/A/d4c7a5c29bd94690bb737decdddd798e 2024-12-03T02:34:21,680 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/78eae6357c1942c8a8c440a544f65e51 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/78eae6357c1942c8a8c440a544f65e51 2024-12-03T02:34:21,681 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/8a74f3ad182d42baab0d28f27698305c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/8a74f3ad182d42baab0d28f27698305c 2024-12-03T02:34:21,682 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/cb170cec1afc420fa398903cb4f7cd04 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/cb170cec1afc420fa398903cb4f7cd04 2024-12-03T02:34:21,683 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/cfa2a0701a564225b13f15c9362a797f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/B/cfa2a0701a564225b13f15c9362a797f 2024-12-03T02:34:21,685 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/173263ecbef048a4851693152b4a263a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/173263ecbef048a4851693152b4a263a 2024-12-03T02:34:21,686 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d222a5af75c47cfaf381957bbc7bc31 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/1d222a5af75c47cfaf381957bbc7bc31 2024-12-03T02:34:21,687 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T02:34:21,687 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/acb960c36a8b44bd9637115aa335eea4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/acb960c36a8b44bd9637115aa335eea4 2024-12-03T02:34:21,688 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/b64fb662e1af4a70a4e319e10ed11bf6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/C/b64fb662e1af4a70a4e319e10ed11bf6 2024-12-03T02:34:21,690 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/recovered.edits/511.seqid to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e/recovered.edits/511.seqid 2024-12-03T02:34:21,691 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,691 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T02:34:21,691 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T02:34:21,692 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T02:34:21,695 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030af4f7a64cd8432681af55f419a8f251_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030af4f7a64cd8432681af55f419a8f251_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,696 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032676ab70eef54d4c87d2b14a2a864374_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032676ab70eef54d4c87d2b14a2a864374_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,697 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032ca2f564fca344b090beceecc66ae12a_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032ca2f564fca344b090beceecc66ae12a_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,699 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120341c22be1521c4c5f91a32c4a74d499fa_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120341c22be1521c4c5f91a32c4a74d499fa_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,700 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034d7cc6c685424a6cbd541708c8ed9d9a_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034d7cc6c685424a6cbd541708c8ed9d9a_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,701 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035407f3db5f384ed99c6b4a531684a739_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035407f3db5f384ed99c6b4a531684a739_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,703 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120358a10f37f7b24c4b8712238dedd33e31_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120358a10f37f7b24c4b8712238dedd33e31_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,704 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120382836340c6a04c108ef9bc8cc2d71987_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120382836340c6a04c108ef9bc8cc2d71987_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,705 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038a0f2ce1ac88404dbe798884919c618e_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038a0f2ce1ac88404dbe798884919c618e_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,706 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120391d55c16e30d4e74ad2fe4ef130593b2_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120391d55c16e30d4e74ad2fe4ef130593b2_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,708 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120394f104b5556b4ead9978cad6cd93fdad_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120394f104b5556b4ead9978cad6cd93fdad_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,709 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039d567313692b45a98d62f4088272092f_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039d567313692b45a98d62f4088272092f_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,710 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a14567124b00415096679991b17cea50_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a14567124b00415096679991b17cea50_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,711 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203afde2d5bb9fd4cb283ba6f111418058e_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203afde2d5bb9fd4cb283ba6f111418058e_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,713 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b7e0e724280c4af89bcfc509f74ff8a1_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b7e0e724280c4af89bcfc509f74ff8a1_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,713 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c24d82fd84f149cb9b27b667e07d18c4_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c24d82fd84f149cb9b27b667e07d18c4_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,714 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c9f5f6e465ee4fe88f7d0e7ff1e79a56_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c9f5f6e465ee4fe88f7d0e7ff1e79a56_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,715 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d2aee8ba38374ad8a5afbbbfab3128c3_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d2aee8ba38374ad8a5afbbbfab3128c3_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,716 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d5b9861e1aa3464bb1ec9fa156885e7c_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d5b9861e1aa3464bb1ec9fa156885e7c_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,717 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203daa4ad15e9ca41cfbc1603aebf5c14a4_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203daa4ad15e9ca41cfbc1603aebf5c14a4_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,718 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e7c9295e8e63497d9542053b29d41cfa_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e7c9295e8e63497d9542053b29d41cfa_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,719 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f084ca92e6ec4d15b867794e88492eef_64446d8651566fb34f3be9578452136e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f084ca92e6ec4d15b867794e88492eef_64446d8651566fb34f3be9578452136e 2024-12-03T02:34:21,719 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T02:34:21,721 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:21,723 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T02:34:21,725 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T02:34:21,726 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:21,726 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T02:34:21,726 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733193261726"}]},"ts":"9223372036854775807"} 2024-12-03T02:34:21,727 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T02:34:21,727 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 64446d8651566fb34f3be9578452136e, NAME => 'TestAcidGuarantees,,1733193232521.64446d8651566fb34f3be9578452136e.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T02:34:21,728 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T02:34:21,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733193261728"}]},"ts":"9223372036854775807"} 2024-12-03T02:34:21,729 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T02:34:21,730 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:21,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-12-03T02:34:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T02:34:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T02:34:21,766 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:21,779 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 249) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2037865029_22 at /127.0.0.1:44306 [Waiting for operation #676] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_756972765_22 at /127.0.0.1:51974 [Waiting for operation #503] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2037865029_22 at /127.0.0.1:46260 [Waiting for operation #669] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=802 (was 913), ProcessCount=11 (was 11), AvailableMemoryMB=2390 (was 2076) - AvailableMemoryMB LEAK? - 2024-12-03T02:34:21,788 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=802, ProcessCount=11, AvailableMemoryMB=2390 2024-12-03T02:34:21,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T02:34:21,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:34:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:21,792 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T02:34:21,792 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:21,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-12-03T02:34:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T02:34:21,793 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T02:34:21,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742056_1232 (size=1061) 2024-12-03T02:34:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T02:34:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T02:34:22,201 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37 2024-12-03T02:34:22,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742057_1233 (size=53) 2024-12-03T02:34:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T02:34:22,607 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:34:22,607 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ee5b914e88815d61556ce041d081a81b, disabling compactions & flushes 2024-12-03T02:34:22,607 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:22,607 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:22,607 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. after waiting 0 ms 2024-12-03T02:34:22,607 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:22,607 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:22,607 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ee5b914e88815d61556ce041d081a81b: Waiting for close lock at 1733193262607Disabling compacts and flushes for region at 1733193262607Disabling writes for close at 1733193262607Writing region close event to WAL at 1733193262607Closed at 1733193262607 2024-12-03T02:34:22,609 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T02:34:22,609 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733193262609"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733193262609"}]},"ts":"1733193262609"} 2024-12-03T02:34:22,611 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T02:34:22,612 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T02:34:22,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193262612"}]},"ts":"1733193262612"} 2024-12-03T02:34:22,614 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T02:34:22,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b, ASSIGN}] 2024-12-03T02:34:22,616 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b, ASSIGN 2024-12-03T02:34:22,616 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b, ASSIGN; state=OFFLINE, location=64035cbb0b46,38721,1733193202350; forceNewPlan=false, retain=false 2024-12-03T02:34:22,767 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=ee5b914e88815d61556ce041d081a81b, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:22,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b, ASSIGN because future has completed 2024-12-03T02:34:22,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:34:22,924 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:22,925 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:34:22,925 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,925 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:34:22,925 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,925 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T02:34:22,927 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,928 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:34:22,929 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5b914e88815d61556ce041d081a81b columnFamilyName A 2024-12-03T02:34:22,929 DEBUG [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:22,930 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] regionserver.HStore(327): Store=ee5b914e88815d61556ce041d081a81b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:34:22,930 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,931 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:34:22,931 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5b914e88815d61556ce041d081a81b columnFamilyName B 2024-12-03T02:34:22,931 DEBUG [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:22,932 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] regionserver.HStore(327): Store=ee5b914e88815d61556ce041d081a81b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:34:22,932 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,933 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:34:22,933 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5b914e88815d61556ce041d081a81b columnFamilyName C 2024-12-03T02:34:22,933 DEBUG [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:22,934 INFO [StoreOpener-ee5b914e88815d61556ce041d081a81b-1 {}] regionserver.HStore(327): Store=ee5b914e88815d61556ce041d081a81b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:34:22,934 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:22,934 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,935 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,936 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,937 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,937 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,938 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:34:22,939 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,944 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:34:22,944 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened ee5b914e88815d61556ce041d081a81b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74567700, jitterRate=0.11114531755447388}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:34:22,944 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:22,945 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for ee5b914e88815d61556ce041d081a81b: Running coprocessor pre-open hook at 1733193262925Writing region info on filesystem at 1733193262925Initializing all the Stores at 1733193262926 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193262927 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193262927Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193262927Cleaning up temporary data from old regions at 1733193262937 (+10 ms)Running coprocessor post-open hooks at 1733193262944 (+7 ms)Region opened successfully at 1733193262945 (+1 ms) 2024-12-03T02:34:22,946 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., pid=57, masterSystemTime=1733193262921 2024-12-03T02:34:22,948 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:22,948 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:22,949 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=ee5b914e88815d61556ce041d081a81b, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:22,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:34:22,951 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44739 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=64035cbb0b46,38721,1733193202350, table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T02:34:22,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-03T02:34:22,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 in 184 msec 2024-12-03T02:34:22,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-03T02:34:22,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b, ASSIGN in 341 msec 2024-12-03T02:34:22,958 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T02:34:22,958 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193262958"}]},"ts":"1733193262958"} 2024-12-03T02:34:22,960 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T02:34:22,961 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T02:34:22,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-12-03T02:34:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T02:34:23,936 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:23,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3260768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f6adac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86a961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,942 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:23,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:23,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:23,946 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T02:34:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 64035cbb0b46,44739,1733193201066 2024-12-03T02:34:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7102cc70 2024-12-03T02:34:23,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T02:34:23,948 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T02:34:23,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555b0fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,950 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261e46b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,952 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9decfe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,955 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733ef71f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,956 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58146a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,960 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34414616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,962 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65558f5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,968 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e95669e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,970 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f31c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,972 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f17cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,975 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c023ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,978 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72427525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,979 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2390ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,983 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c57cde3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,985 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57264a94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,989 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0de7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0495b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,994 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:23,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742c620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:23,997 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d023a14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:23,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:23,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:24,000 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5383b7b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:24,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:24,002 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fb1bd3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:24,005 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,005 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,006 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:24,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-03T02:34:24,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T02:34:24,009 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:24,010 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,010 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,010 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:24,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,011 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,012 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,013 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,014 DEBUG [Thread-1094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,015 DEBUG [Thread-1096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,016 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:24,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:24,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:34:24,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:34:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:24,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/9ca39eda597c422e83de1103fb9b1d84 is 50, key is test_row_0/A:col10/1733193264041/Put/seqid=0 2024-12-03T02:34:24,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193264096, max timestamp=1733193264110Num uniques 859; 2024-12-03T02:34:24,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T02:34:24,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193264096, max timestamp=1733193264110Num uniques 859; 2024-12-03T02:34:24,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193264096, max timestamp=1733193264110Num uniques 859; 2024-12-03T02:34:24,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:24,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:24,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742058_1234 (size=12001) 2024-12-03T02:34:24,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/9ca39eda597c422e83de1103fb9b1d84 2024-12-03T02:34:24,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193324148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193324149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193324153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193324153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193324154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T02:34:24,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:24,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/36067541f3544a588c56b7cf4f1361f2 is 50, key is test_row_0/B:col10/1733193264041/Put/seqid=0 2024-12-03T02:34:24,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742059_1235 (size=12001) 2024-12-03T02:34:24,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193324265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193324265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193324275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193324275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193324276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T02:34:24,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T02:34:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193324475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193324475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T02:34:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193324485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193324485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193324487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T02:34:24,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T02:34:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/36067541f3544a588c56b7cf4f1361f2 2024-12-03T02:34:24,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/069f6326c077473aaa28568b68f727df is 50, key is test_row_0/C:col10/1733193264041/Put/seqid=0 2024-12-03T02:34:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742060_1236 (size=12001) 2024-12-03T02:34:24,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193324784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193324785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T02:34:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193324795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193324795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193324805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:24,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T02:34:24,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:24,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:25,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/069f6326c077473aaa28568b68f727df 2024-12-03T02:34:25,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/9ca39eda597c422e83de1103fb9b1d84 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9ca39eda597c422e83de1103fb9b1d84 2024-12-03T02:34:25,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9ca39eda597c422e83de1103fb9b1d84, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T02:34:25,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/36067541f3544a588c56b7cf4f1361f2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/36067541f3544a588c56b7cf4f1361f2 2024-12-03T02:34:25,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T02:34:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:25,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:25,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/36067541f3544a588c56b7cf4f1361f2, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T02:34:25,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/069f6326c077473aaa28568b68f727df as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/069f6326c077473aaa28568b68f727df 2024-12-03T02:34:25,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/069f6326c077473aaa28568b68f727df, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T02:34:25,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ee5b914e88815d61556ce041d081a81b in 1069ms, sequenceid=12, compaction requested=false 2024-12-03T02:34:25,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T02:34:25,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T02:34:25,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:25,254 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-03T02:34:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:25,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/18c68261268b46a8aba8d457f572b700 is 50, key is test_row_0/A:col10/1733193264152/Put/seqid=0 2024-12-03T02:34:25,267 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:34:25,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742061_1237 (size=12001) 2024-12-03T02:34:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:25,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:25,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193325308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193325309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193325309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193325311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193325315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193325415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193325424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193325425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193325425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193325624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193325635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193325635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193325635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,675 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/18c68261268b46a8aba8d457f572b700 2024-12-03T02:34:25,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a9f9e73b43a745a6b7bfb5d3362100dc is 50, key is test_row_0/B:col10/1733193264152/Put/seqid=0 2024-12-03T02:34:25,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742062_1238 (size=12001) 2024-12-03T02:34:25,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193325935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193325945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193325945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193325945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:26,144 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a9f9e73b43a745a6b7bfb5d3362100dc 2024-12-03T02:34:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/cdfa7dafbd754eef87b0eb43cb0f3301 is 50, key is test_row_0/C:col10/1733193264152/Put/seqid=0 2024-12-03T02:34:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T02:34:26,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742063_1239 (size=12001) 2024-12-03T02:34:26,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193326325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:26,327 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:26,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193326445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193326454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193326454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193326454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:26,571 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/cdfa7dafbd754eef87b0eb43cb0f3301 2024-12-03T02:34:26,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/18c68261268b46a8aba8d457f572b700 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/18c68261268b46a8aba8d457f572b700 2024-12-03T02:34:26,597 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/18c68261268b46a8aba8d457f572b700, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T02:34:26,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a9f9e73b43a745a6b7bfb5d3362100dc as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a9f9e73b43a745a6b7bfb5d3362100dc 2024-12-03T02:34:26,604 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a9f9e73b43a745a6b7bfb5d3362100dc, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T02:34:26,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/cdfa7dafbd754eef87b0eb43cb0f3301 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/cdfa7dafbd754eef87b0eb43cb0f3301 2024-12-03T02:34:26,611 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/cdfa7dafbd754eef87b0eb43cb0f3301, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T02:34:26,612 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ee5b914e88815d61556ce041d081a81b in 1358ms, sequenceid=47, compaction requested=false 2024-12-03T02:34:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:26,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-03T02:34:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-03T02:34:26,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-03T02:34:26,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6040 sec 2024-12-03T02:34:26,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.6110 sec 2024-12-03T02:34:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:27,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T02:34:27,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:27,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:27,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:27,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:27,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:27,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:27,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/ada7832e135f4649bebfc2a93d965274 is 50, key is test_row_0/A:col10/1733193265301/Put/seqid=0 2024-12-03T02:34:27,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742064_1240 (size=12001) 2024-12-03T02:34:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193327490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193327490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193327492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193327494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193327605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193327605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193327605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193327605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193327815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193327815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193327815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193327815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:27,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:27,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/ada7832e135f4649bebfc2a93d965274 2024-12-03T02:34:27,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/766ea15750c0464bafb478c9a6e6b45c is 50, key is test_row_0/B:col10/1733193265301/Put/seqid=0 2024-12-03T02:34:27,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742065_1241 (size=12001) 2024-12-03T02:34:28,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193328125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:28,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193328125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193328125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:28,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193328125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T02:34:28,175 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:28,175 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-03T02:34:28,177 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T02:34:28,180 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:28,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T02:34:28,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T02:34:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/766ea15750c0464bafb478c9a6e6b45c 2024-12-03T02:34:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193328346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:28,347 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:28,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/95ccd58bae344a6daa7068f50f7d97a6 is 50, key is test_row_0/C:col10/1733193265301/Put/seqid=0 2024-12-03T02:34:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742066_1242 (size=12001) 2024-12-03T02:34:28,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T02:34:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T02:34:28,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193328635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193328635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193328635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193328635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:28,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T02:34:28,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:28,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:28,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/95ccd58bae344a6daa7068f50f7d97a6 2024-12-03T02:34:28,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/ada7832e135f4649bebfc2a93d965274 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/ada7832e135f4649bebfc2a93d965274 2024-12-03T02:34:28,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/ada7832e135f4649bebfc2a93d965274, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T02:34:28,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/766ea15750c0464bafb478c9a6e6b45c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/766ea15750c0464bafb478c9a6e6b45c 2024-12-03T02:34:28,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/766ea15750c0464bafb478c9a6e6b45c, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T02:34:28,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/95ccd58bae344a6daa7068f50f7d97a6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/95ccd58bae344a6daa7068f50f7d97a6 2024-12-03T02:34:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T02:34:28,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T02:34:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:28,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/95ccd58bae344a6daa7068f50f7d97a6, entries=150, sequenceid=59, filesize=11.7 K 2024-12-03T02:34:28,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ee5b914e88815d61556ce041d081a81b in 1356ms, sequenceid=59, compaction requested=true 2024-12-03T02:34:28,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:28,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:28,817 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:28,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:28,817 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:28,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:28,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:28,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:28,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:28,819 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:28,819 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/A is initiating minor compaction (all files) 2024-12-03T02:34:28,819 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:28,819 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/A in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,819 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/B is initiating minor compaction (all files) 2024-12-03T02:34:28,819 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/B in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,819 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9ca39eda597c422e83de1103fb9b1d84, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/18c68261268b46a8aba8d457f572b700, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/ada7832e135f4649bebfc2a93d965274] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=35.2 K 2024-12-03T02:34:28,819 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/36067541f3544a588c56b7cf4f1361f2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a9f9e73b43a745a6b7bfb5d3362100dc, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/766ea15750c0464bafb478c9a6e6b45c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=35.2 K 2024-12-03T02:34:28,820 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca39eda597c422e83de1103fb9b1d84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733193264041 2024-12-03T02:34:28,820 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 36067541f3544a588c56b7cf4f1361f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733193264041 2024-12-03T02:34:28,820 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting a9f9e73b43a745a6b7bfb5d3362100dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733193264134 2024-12-03T02:34:28,820 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18c68261268b46a8aba8d457f572b700, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733193264134 2024-12-03T02:34:28,820 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 766ea15750c0464bafb478c9a6e6b45c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733193265301 2024-12-03T02:34:28,820 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ada7832e135f4649bebfc2a93d965274, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733193265301 2024-12-03T02:34:28,831 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:28,831 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/21db5e2714d6431c83b5a632acd7c347 is 50, key is test_row_0/B:col10/1733193265301/Put/seqid=0 2024-12-03T02:34:28,843 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#A#compaction#196 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:28,845 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/bb33555f96ce47d7a7b2cd69f82c625d is 50, key is test_row_0/A:col10/1733193265301/Put/seqid=0 2024-12-03T02:34:28,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742067_1243 (size=12104) 2024-12-03T02:34:28,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742068_1244 (size=12104) 2024-12-03T02:34:28,872 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/21db5e2714d6431c83b5a632acd7c347 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21db5e2714d6431c83b5a632acd7c347 2024-12-03T02:34:28,880 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/B of ee5b914e88815d61556ce041d081a81b into 21db5e2714d6431c83b5a632acd7c347(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:28,880 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:28,880 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/B, priority=13, startTime=1733193268817; duration=0sec 2024-12-03T02:34:28,880 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:28,880 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:B 2024-12-03T02:34:28,881 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:28,882 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:28,882 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/C is initiating minor compaction (all files) 2024-12-03T02:34:28,883 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/C in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,883 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/069f6326c077473aaa28568b68f727df, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/cdfa7dafbd754eef87b0eb43cb0f3301, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/95ccd58bae344a6daa7068f50f7d97a6] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=35.2 K 2024-12-03T02:34:28,884 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 069f6326c077473aaa28568b68f727df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733193264041 2024-12-03T02:34:28,884 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting cdfa7dafbd754eef87b0eb43cb0f3301, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733193264134 2024-12-03T02:34:28,885 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ccd58bae344a6daa7068f50f7d97a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733193265301 2024-12-03T02:34:28,899 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:28,899 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/417df89672c54b209bbfd0901f690fda is 50, key is test_row_0/C:col10/1733193265301/Put/seqid=0 2024-12-03T02:34:28,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742069_1245 (size=12104) 2024-12-03T02:34:28,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T02:34:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:28,965 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T02:34:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:28,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:28,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/e93311d550d9471d92d1d78ca0d1699b is 50, key is test_row_0/A:col10/1733193267485/Put/seqid=0 2024-12-03T02:34:28,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742070_1246 (size=12001) 2024-12-03T02:34:28,987 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/e93311d550d9471d92d1d78ca0d1699b 2024-12-03T02:34:28,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/fec5f9b8d213452eb8bab359a8ef359b is 50, key is test_row_0/B:col10/1733193267485/Put/seqid=0 2024-12-03T02:34:29,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742071_1247 (size=12001) 2024-12-03T02:34:29,278 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/bb33555f96ce47d7a7b2cd69f82c625d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bb33555f96ce47d7a7b2cd69f82c625d 2024-12-03T02:34:29,283 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/A of ee5b914e88815d61556ce041d081a81b into bb33555f96ce47d7a7b2cd69f82c625d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:29,284 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:29,284 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/A, priority=13, startTime=1733193268817; duration=0sec 2024-12-03T02:34:29,284 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:29,284 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:A 2024-12-03T02:34:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T02:34:29,332 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/417df89672c54b209bbfd0901f690fda as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/417df89672c54b209bbfd0901f690fda 2024-12-03T02:34:29,336 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/C of ee5b914e88815d61556ce041d081a81b into 417df89672c54b209bbfd0901f690fda(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:29,337 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:29,337 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/C, priority=13, startTime=1733193268818; duration=0sec 2024-12-03T02:34:29,337 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:29,337 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:C 2024-12-03T02:34:29,412 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/fec5f9b8d213452eb8bab359a8ef359b 2024-12-03T02:34:29,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/0c23833558304fa18f389e57c0f1b841 is 50, key is test_row_0/C:col10/1733193267485/Put/seqid=0 2024-12-03T02:34:29,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742072_1248 (size=12001) 2024-12-03T02:34:29,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193329663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193329665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193329665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193329665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:29,667 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193329775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193329775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193329775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:29,854 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/0c23833558304fa18f389e57c0f1b841 2024-12-03T02:34:29,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/e93311d550d9471d92d1d78ca0d1699b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e93311d550d9471d92d1d78ca0d1699b 2024-12-03T02:34:29,868 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e93311d550d9471d92d1d78ca0d1699b, entries=150, sequenceid=80, filesize=11.7 K 2024-12-03T02:34:29,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/fec5f9b8d213452eb8bab359a8ef359b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/fec5f9b8d213452eb8bab359a8ef359b 2024-12-03T02:34:29,873 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/fec5f9b8d213452eb8bab359a8ef359b, entries=150, sequenceid=80, filesize=11.7 K 2024-12-03T02:34:29,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/0c23833558304fa18f389e57c0f1b841 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0c23833558304fa18f389e57c0f1b841 2024-12-03T02:34:29,879 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0c23833558304fa18f389e57c0f1b841, entries=150, sequenceid=80, filesize=11.7 K 2024-12-03T02:34:29,880 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ee5b914e88815d61556ce041d081a81b in 914ms, sequenceid=80, compaction requested=false 2024-12-03T02:34:29,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:29,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:29,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-03T02:34:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-03T02:34:29,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T02:34:29,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7010 sec 2024-12-03T02:34:29,886 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.7100 sec 2024-12-03T02:34:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:29,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T02:34:29,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:29,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:29,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:29,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:30,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/bc523a35fb844afb90c528b274dc2d3a is 50, key is test_row_0/A:col10/1733193269663/Put/seqid=0 2024-12-03T02:34:30,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742073_1249 (size=12001) 2024-12-03T02:34:30,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193330022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193330025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/bc523a35fb844afb90c528b274dc2d3a 2024-12-03T02:34:30,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193330029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a4c2d17bd3a0464cbdcbd8023c8ef7a4 is 50, key is test_row_0/B:col10/1733193269663/Put/seqid=0 2024-12-03T02:34:30,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742074_1250 (size=12001) 2024-12-03T02:34:30,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a4c2d17bd3a0464cbdcbd8023c8ef7a4 2024-12-03T02:34:30,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/b1acee84cb9948438b16a3cbfb2b3051 is 50, key is test_row_0/C:col10/1733193269663/Put/seqid=0 2024-12-03T02:34:30,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193330139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193330139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193330145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742075_1251 (size=12001) 2024-12-03T02:34:30,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/b1acee84cb9948438b16a3cbfb2b3051 2024-12-03T02:34:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/bc523a35fb844afb90c528b274dc2d3a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bc523a35fb844afb90c528b274dc2d3a 2024-12-03T02:34:30,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bc523a35fb844afb90c528b274dc2d3a, entries=150, sequenceid=97, filesize=11.7 K 2024-12-03T02:34:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a4c2d17bd3a0464cbdcbd8023c8ef7a4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a4c2d17bd3a0464cbdcbd8023c8ef7a4 2024-12-03T02:34:30,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a4c2d17bd3a0464cbdcbd8023c8ef7a4, entries=150, sequenceid=97, filesize=11.7 K 2024-12-03T02:34:30,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/b1acee84cb9948438b16a3cbfb2b3051 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/b1acee84cb9948438b16a3cbfb2b3051 2024-12-03T02:34:30,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/b1acee84cb9948438b16a3cbfb2b3051, entries=150, sequenceid=97, filesize=11.7 K 2024-12-03T02:34:30,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ee5b914e88815d61556ce041d081a81b in 241ms, sequenceid=97, compaction requested=true 2024-12-03T02:34:30,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:30,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:30,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:30,228 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:30,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:30,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:30,228 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:30,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:30,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:30,229 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:30,229 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/B is initiating minor compaction (all files) 2024-12-03T02:34:30,229 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/B in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,229 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21db5e2714d6431c83b5a632acd7c347, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/fec5f9b8d213452eb8bab359a8ef359b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a4c2d17bd3a0464cbdcbd8023c8ef7a4] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=35.3 K 2024-12-03T02:34:30,229 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:30,229 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/A is initiating minor compaction (all files) 2024-12-03T02:34:30,229 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/A in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,230 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bb33555f96ce47d7a7b2cd69f82c625d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e93311d550d9471d92d1d78ca0d1699b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bc523a35fb844afb90c528b274dc2d3a] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=35.3 K 2024-12-03T02:34:30,230 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb33555f96ce47d7a7b2cd69f82c625d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733193265301 2024-12-03T02:34:30,230 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 21db5e2714d6431c83b5a632acd7c347, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733193265301 2024-12-03T02:34:30,230 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting fec5f9b8d213452eb8bab359a8ef359b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733193267485 2024-12-03T02:34:30,231 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e93311d550d9471d92d1d78ca0d1699b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733193267485 2024-12-03T02:34:30,231 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting a4c2d17bd3a0464cbdcbd8023c8ef7a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733193269663 2024-12-03T02:34:30,231 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc523a35fb844afb90c528b274dc2d3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733193269663 2024-12-03T02:34:30,245 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:30,246 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/709eae36d4114ba5a2e246adfa8ee157 is 50, key is test_row_0/A:col10/1733193269663/Put/seqid=0 2024-12-03T02:34:30,248 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#B#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:30,249 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/eb886812051c4594a28c225f495cc348 is 50, key is test_row_0/B:col10/1733193269663/Put/seqid=0 2024-12-03T02:34:30,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742076_1252 (size=12207) 2024-12-03T02:34:30,285 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/709eae36d4114ba5a2e246adfa8ee157 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/709eae36d4114ba5a2e246adfa8ee157 2024-12-03T02:34:30,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742077_1253 (size=12207) 2024-12-03T02:34:30,293 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/A of ee5b914e88815d61556ce041d081a81b into 709eae36d4114ba5a2e246adfa8ee157(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:30,293 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:30,293 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/A, priority=13, startTime=1733193270228; duration=0sec 2024-12-03T02:34:30,294 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:30,294 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:A 2024-12-03T02:34:30,294 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:30,296 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:30,296 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/C is initiating minor compaction (all files) 2024-12-03T02:34:30,296 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/C in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,296 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/417df89672c54b209bbfd0901f690fda, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0c23833558304fa18f389e57c0f1b841, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/b1acee84cb9948438b16a3cbfb2b3051] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=35.3 K 2024-12-03T02:34:30,298 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417df89672c54b209bbfd0901f690fda, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733193265301 2024-12-03T02:34:30,298 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c23833558304fa18f389e57c0f1b841, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733193267485 2024-12-03T02:34:30,298 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1acee84cb9948438b16a3cbfb2b3051, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733193269663 2024-12-03T02:34:30,303 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/eb886812051c4594a28c225f495cc348 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/eb886812051c4594a28c225f495cc348 2024-12-03T02:34:30,310 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/B of ee5b914e88815d61556ce041d081a81b into eb886812051c4594a28c225f495cc348(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:30,310 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:30,310 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/B, priority=13, startTime=1733193270228; duration=0sec 2024-12-03T02:34:30,310 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:30,310 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:B 2024-12-03T02:34:30,313 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:30,313 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/a3c77796348d41b0b6a6fb69c5be824f is 50, key is test_row_0/C:col10/1733193269663/Put/seqid=0 2024-12-03T02:34:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T02:34:30,325 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:30,326 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-03T02:34:30,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T02:34:30,329 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:30,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:30,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742078_1254 (size=12207) 2024-12-03T02:34:30,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T02:34:30,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:30,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:30,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:30,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:30,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:30,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:30,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/2207f5bb069e472da509c172de1ada04 is 50, key is test_row_0/A:col10/1733193270356/Put/seqid=0 2024-12-03T02:34:30,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193330390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193330393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193330402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742079_1255 (size=12001) 2024-12-03T02:34:30,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/2207f5bb069e472da509c172de1ada04 2024-12-03T02:34:30,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/9c945e55cec0400497d59d0ef575655d is 50, key is test_row_0/B:col10/1733193270356/Put/seqid=0 2024-12-03T02:34:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T02:34:30,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742080_1256 (size=12001) 2024-12-03T02:34:30,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T02:34:30,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:30,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193330505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193330505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193330514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T02:34:30,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:30,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T02:34:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193330714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193330715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193330725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:30,744 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/a3c77796348d41b0b6a6fb69c5be824f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/a3c77796348d41b0b6a6fb69c5be824f 2024-12-03T02:34:30,752 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/C of ee5b914e88815d61556ce041d081a81b into a3c77796348d41b0b6a6fb69c5be824f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:30,752 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:30,752 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/C, priority=13, startTime=1733193270228; duration=0sec 2024-12-03T02:34:30,752 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:30,752 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:C 2024-12-03T02:34:30,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T02:34:30,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:30,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/9c945e55cec0400497d59d0ef575655d 2024-12-03T02:34:30,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/1f6e33234b0148fbb105bc6f9abd9ed9 is 50, key is test_row_0/C:col10/1733193270356/Put/seqid=0 2024-12-03T02:34:30,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742081_1257 (size=12001) 2024-12-03T02:34:30,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T02:34:30,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:30,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:30,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T02:34:31,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193331024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193331025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193331035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T02:34:31,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:31,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:31,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:31,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:31,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:31,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T02:34:31,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:31,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:31,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:31,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:31,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:31,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/1f6e33234b0148fbb105bc6f9abd9ed9 2024-12-03T02:34:31,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/2207f5bb069e472da509c172de1ada04 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2207f5bb069e472da509c172de1ada04 2024-12-03T02:34:31,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2207f5bb069e472da509c172de1ada04, entries=150, sequenceid=117, filesize=11.7 K 2024-12-03T02:34:31,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/9c945e55cec0400497d59d0ef575655d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9c945e55cec0400497d59d0ef575655d 2024-12-03T02:34:31,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9c945e55cec0400497d59d0ef575655d, entries=150, sequenceid=117, filesize=11.7 K 2024-12-03T02:34:31,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/1f6e33234b0148fbb105bc6f9abd9ed9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1f6e33234b0148fbb105bc6f9abd9ed9 2024-12-03T02:34:31,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1f6e33234b0148fbb105bc6f9abd9ed9, entries=150, sequenceid=117, filesize=11.7 K 2024-12-03T02:34:31,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ee5b914e88815d61556ce041d081a81b in 950ms, sequenceid=117, compaction requested=false 2024-12-03T02:34:31,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:31,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T02:34:31,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:31,410 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T02:34:31,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:31,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:31,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:31,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:31,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:31,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:31,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/be721cd2f0034229889350c762403daf is 50, key is test_row_0/A:col10/1733193270380/Put/seqid=0 2024-12-03T02:34:31,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742082_1258 (size=12101) 2024-12-03T02:34:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T02:34:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:31,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:31,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193331567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193331570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193331571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193331674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193331674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,677 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,687 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T02:34:31,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T02:34:31,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193331687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193331690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,822 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/be721cd2f0034229889350c762403daf 2024-12-03T02:34:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/7bb53f21befd44749f0e53b2d3d0a177 is 50, key is test_row_0/B:col10/1733193270380/Put/seqid=0 2024-12-03T02:34:31,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742083_1259 (size=12101) 2024-12-03T02:34:31,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193331884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193331895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:31,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193331895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:31,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:31,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:31,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193332194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193332205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:32,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:32,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193332205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,254 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/7bb53f21befd44749f0e53b2d3d0a177 2024-12-03T02:34:32,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/7c300bd631ed4953afd0869583a7706f is 50, key is test_row_0/C:col10/1733193270380/Put/seqid=0 2024-12-03T02:34:32,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742084_1260 (size=12101) 2024-12-03T02:34:32,273 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/7c300bd631ed4953afd0869583a7706f 2024-12-03T02:34:32,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/be721cd2f0034229889350c762403daf as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/be721cd2f0034229889350c762403daf 2024-12-03T02:34:32,283 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/be721cd2f0034229889350c762403daf, entries=150, sequenceid=132, filesize=11.8 K 2024-12-03T02:34:32,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/7bb53f21befd44749f0e53b2d3d0a177 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/7bb53f21befd44749f0e53b2d3d0a177 2024-12-03T02:34:32,288 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/7bb53f21befd44749f0e53b2d3d0a177, entries=150, sequenceid=132, filesize=11.8 K 2024-12-03T02:34:32,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/7c300bd631ed4953afd0869583a7706f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7c300bd631ed4953afd0869583a7706f 2024-12-03T02:34:32,293 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7c300bd631ed4953afd0869583a7706f, entries=150, sequenceid=132, filesize=11.8 K 2024-12-03T02:34:32,294 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ee5b914e88815d61556ce041d081a81b in 884ms, sequenceid=132, compaction requested=true 2024-12-03T02:34:32,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:32,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:32,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-03T02:34:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-03T02:34:32,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-03T02:34:32,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9660 sec 2024-12-03T02:34:32,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.9730 sec 2024-12-03T02:34:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:32,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T02:34:32,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:32,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:32,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:32,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:32,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:32,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:32,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/a1137e7f84f34ceda038fb65d3b98f89 is 50, key is test_row_0/A:col10/1733193272375/Put/seqid=0 2024-12-03T02:34:32,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742085_1261 (size=14541) 2024-12-03T02:34:32,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193332410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T02:34:32,489 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:32,493 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-03T02:34:32,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T02:34:32,501 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:32,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:32,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193332524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T02:34:32,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:32,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:32,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:32,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193332705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:32,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193332714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193332714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193332735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:32,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/a1137e7f84f34ceda038fb65d3b98f89 2024-12-03T02:34:32,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/89d843fd679e4299a7b5c23c1f3f5c2f is 50, key is test_row_0/B:col10/1733193272375/Put/seqid=0 2024-12-03T02:34:32,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742086_1262 (size=12151) 2024-12-03T02:34:32,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:32,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:32,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:32,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T02:34:32,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:32,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:32,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:32,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:32,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:32,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193333044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:33,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:33,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T02:34:33,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/89d843fd679e4299a7b5c23c1f3f5c2f 2024-12-03T02:34:33,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/0f65416c93604a7fbe1088ee287a6a88 is 50, key is test_row_0/C:col10/1733193272375/Put/seqid=0 2024-12-03T02:34:33,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742087_1263 (size=12151) 2024-12-03T02:34:33,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193333555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:33,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:33,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:33,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/0f65416c93604a7fbe1088ee287a6a88 2024-12-03T02:34:33,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/a1137e7f84f34ceda038fb65d3b98f89 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a1137e7f84f34ceda038fb65d3b98f89 2024-12-03T02:34:33,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a1137e7f84f34ceda038fb65d3b98f89, entries=200, sequenceid=151, filesize=14.2 K 2024-12-03T02:34:33,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/89d843fd679e4299a7b5c23c1f3f5c2f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/89d843fd679e4299a7b5c23c1f3f5c2f 2024-12-03T02:34:33,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/89d843fd679e4299a7b5c23c1f3f5c2f, entries=150, sequenceid=151, filesize=11.9 K 2024-12-03T02:34:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T02:34:33,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/0f65416c93604a7fbe1088ee287a6a88 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0f65416c93604a7fbe1088ee287a6a88 2024-12-03T02:34:33,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0f65416c93604a7fbe1088ee287a6a88, entries=150, sequenceid=151, filesize=11.9 K 2024-12-03T02:34:33,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ee5b914e88815d61556ce041d081a81b in 1270ms, sequenceid=151, compaction requested=true 2024-12-03T02:34:33,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:33,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:33,647 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:33,647 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:33,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:33,648 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:33,648 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/B is initiating minor compaction (all files) 2024-12-03T02:34:33,648 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/B in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,648 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/eb886812051c4594a28c225f495cc348, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9c945e55cec0400497d59d0ef575655d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/7bb53f21befd44749f0e53b2d3d0a177, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/89d843fd679e4299a7b5c23c1f3f5c2f] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=47.3 K 2024-12-03T02:34:33,648 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:33,648 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/A is initiating minor compaction (all files) 2024-12-03T02:34:33,649 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/A in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,649 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/709eae36d4114ba5a2e246adfa8ee157, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2207f5bb069e472da509c172de1ada04, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/be721cd2f0034229889350c762403daf, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a1137e7f84f34ceda038fb65d3b98f89] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=49.7 K 2024-12-03T02:34:33,649 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting eb886812051c4594a28c225f495cc348, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733193269663 2024-12-03T02:34:33,649 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 709eae36d4114ba5a2e246adfa8ee157, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733193269663 2024-12-03T02:34:33,649 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c945e55cec0400497d59d0ef575655d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733193270016 2024-12-03T02:34:33,650 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2207f5bb069e472da509c172de1ada04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733193270016 2024-12-03T02:34:33,651 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bb53f21befd44749f0e53b2d3d0a177, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733193270380 2024-12-03T02:34:33,651 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting be721cd2f0034229889350c762403daf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733193270380 2024-12-03T02:34:33,652 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 89d843fd679e4299a7b5c23c1f3f5c2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733193271562 2024-12-03T02:34:33,652 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1137e7f84f34ceda038fb65d3b98f89, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733193271562 2024-12-03T02:34:33,663 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:33,663 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/a802042ed56e44738a697b7d4c05263d is 50, key is test_row_0/A:col10/1733193272375/Put/seqid=0 2024-12-03T02:34:33,665 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:33,665 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/12f40b2f19034ca5b2b1c78cfe2bd687 is 50, key is test_row_0/B:col10/1733193272375/Put/seqid=0 2024-12-03T02:34:33,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742089_1265 (size=12493) 2024-12-03T02:34:33,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742088_1264 (size=12493) 2024-12-03T02:34:33,686 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/12f40b2f19034ca5b2b1c78cfe2bd687 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/12f40b2f19034ca5b2b1c78cfe2bd687 2024-12-03T02:34:33,687 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/a802042ed56e44738a697b7d4c05263d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a802042ed56e44738a697b7d4c05263d 2024-12-03T02:34:33,690 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee5b914e88815d61556ce041d081a81b/B of ee5b914e88815d61556ce041d081a81b into 12f40b2f19034ca5b2b1c78cfe2bd687(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:33,690 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:33,690 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/B, priority=12, startTime=1733193273647; duration=0sec 2024-12-03T02:34:33,690 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:33,690 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:B 2024-12-03T02:34:33,690 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:33,692 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:33,692 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/C is initiating minor compaction (all files) 2024-12-03T02:34:33,692 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/C in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,693 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/a3c77796348d41b0b6a6fb69c5be824f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1f6e33234b0148fbb105bc6f9abd9ed9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7c300bd631ed4953afd0869583a7706f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0f65416c93604a7fbe1088ee287a6a88] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=47.3 K 2024-12-03T02:34:33,693 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting a3c77796348d41b0b6a6fb69c5be824f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733193269663 2024-12-03T02:34:33,693 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f6e33234b0148fbb105bc6f9abd9ed9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733193270016 2024-12-03T02:34:33,693 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c300bd631ed4953afd0869583a7706f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733193270380 2024-12-03T02:34:33,694 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f65416c93604a7fbe1088ee287a6a88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733193271562 2024-12-03T02:34:33,694 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee5b914e88815d61556ce041d081a81b/A of ee5b914e88815d61556ce041d081a81b into a802042ed56e44738a697b7d4c05263d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:33,695 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:33,695 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/A, priority=12, startTime=1733193273646; duration=0sec 2024-12-03T02:34:33,695 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:33,695 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:A 2024-12-03T02:34:33,704 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:33,704 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/aaecb1bb12694509b88f66d5dd94344f is 50, key is test_row_0/C:col10/1733193272375/Put/seqid=0 2024-12-03T02:34:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:33,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742090_1266 (size=12493) 2024-12-03T02:34:33,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T02:34:33,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:33,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:33,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:33,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:33,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:33,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:33,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/591cd13fac8d4a94ba54cc85f94b8045 is 50, key is test_row_0/A:col10/1733193272392/Put/seqid=0 2024-12-03T02:34:33,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:33,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:33,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742091_1267 (size=12151) 2024-12-03T02:34:33,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193333759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:33,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193333760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:33,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193333763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:33,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:33,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193333875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:33,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193333875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193333875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:33,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:33,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193334085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193334085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193334085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/591cd13fac8d4a94ba54cc85f94b8045 2024-12-03T02:34:34,146 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/aaecb1bb12694509b88f66d5dd94344f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/aaecb1bb12694509b88f66d5dd94344f 2024-12-03T02:34:34,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/64142b41cda74167850bbe779ef2705e is 50, key is test_row_0/B:col10/1733193272392/Put/seqid=0 2024-12-03T02:34:34,155 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee5b914e88815d61556ce041d081a81b/C of ee5b914e88815d61556ce041d081a81b into aaecb1bb12694509b88f66d5dd94344f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:34,155 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:34,155 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/C, priority=12, startTime=1733193273647; duration=0sec 2024-12-03T02:34:34,155 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:34,155 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:C 2024-12-03T02:34:34,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742092_1268 (size=12151) 2024-12-03T02:34:34,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:34,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:34,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:34,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:34,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193334395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193334395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193334395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/64142b41cda74167850bbe779ef2705e 2024-12-03T02:34:34,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/975466afe8f443bf9c8165648c7d11cf is 50, key is test_row_0/C:col10/1733193272392/Put/seqid=0 2024-12-03T02:34:34,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193334564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,567 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:34,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742093_1269 (size=12151) 2024-12-03T02:34:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T02:34:34,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:34,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193334905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193334905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193334905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:34,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:34,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:34,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:34,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/975466afe8f443bf9c8165648c7d11cf 2024-12-03T02:34:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/591cd13fac8d4a94ba54cc85f94b8045 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/591cd13fac8d4a94ba54cc85f94b8045 2024-12-03T02:34:34,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/591cd13fac8d4a94ba54cc85f94b8045, entries=150, sequenceid=168, filesize=11.9 K 2024-12-03T02:34:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/64142b41cda74167850bbe779ef2705e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/64142b41cda74167850bbe779ef2705e 2024-12-03T02:34:34,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/64142b41cda74167850bbe779ef2705e, entries=150, sequenceid=168, filesize=11.9 K 2024-12-03T02:34:34,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/975466afe8f443bf9c8165648c7d11cf as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/975466afe8f443bf9c8165648c7d11cf 2024-12-03T02:34:34,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/975466afe8f443bf9c8165648c7d11cf, entries=150, sequenceid=168, filesize=11.9 K 2024-12-03T02:34:34,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ee5b914e88815d61556ce041d081a81b in 1268ms, sequenceid=168, compaction requested=false 2024-12-03T02:34:34,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:35,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T02:34:35,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:35,121 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T02:34:35,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:35,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:35,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:35,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:35,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:35,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:35,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/bbb493b710fa4c5a80cc807f8da7e1b6 is 50, key is test_row_0/A:col10/1733193273762/Put/seqid=0 2024-12-03T02:34:35,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742094_1270 (size=12151) 2024-12-03T02:34:35,540 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/bbb493b710fa4c5a80cc807f8da7e1b6 2024-12-03T02:34:35,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/e33551509f2544ac8ff6e2569db1f75e is 50, key is test_row_0/B:col10/1733193273762/Put/seqid=0 2024-12-03T02:34:35,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742095_1271 (size=12151) 2024-12-03T02:34:35,565 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/e33551509f2544ac8ff6e2569db1f75e 2024-12-03T02:34:35,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/1eb53a6d73ff436da1a69e7b495d58f4 is 50, key is test_row_0/C:col10/1733193273762/Put/seqid=0 2024-12-03T02:34:35,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742096_1272 (size=12151) 2024-12-03T02:34:35,577 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/1eb53a6d73ff436da1a69e7b495d58f4 2024-12-03T02:34:35,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/bbb493b710fa4c5a80cc807f8da7e1b6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bbb493b710fa4c5a80cc807f8da7e1b6 2024-12-03T02:34:35,586 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bbb493b710fa4c5a80cc807f8da7e1b6, entries=150, sequenceid=186, filesize=11.9 K 2024-12-03T02:34:35,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/e33551509f2544ac8ff6e2569db1f75e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/e33551509f2544ac8ff6e2569db1f75e 2024-12-03T02:34:35,590 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/e33551509f2544ac8ff6e2569db1f75e, entries=150, sequenceid=186, filesize=11.9 K 2024-12-03T02:34:35,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/1eb53a6d73ff436da1a69e7b495d58f4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1eb53a6d73ff436da1a69e7b495d58f4 2024-12-03T02:34:35,595 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1eb53a6d73ff436da1a69e7b495d58f4, entries=150, sequenceid=186, filesize=11.9 K 2024-12-03T02:34:35,595 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for ee5b914e88815d61556ce041d081a81b in 474ms, sequenceid=186, compaction requested=true 2024-12-03T02:34:35,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:35,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:35,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-03T02:34:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-03T02:34:35,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-03T02:34:35,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0960 sec 2024-12-03T02:34:35,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 3.1060 sec 2024-12-03T02:34:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:35,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:34:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:35,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:35,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:35,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/fa1cc4540d0949c49fc1566bed07ea3c is 50, key is test_row_0/A:col10/1733193275699/Put/seqid=0 2024-12-03T02:34:35,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742097_1273 (size=14541) 2024-12-03T02:34:35,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193275745, max timestamp=1733193275774Num uniques 859; 2024-12-03T02:34:35,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193275745, max timestamp=1733193275774Num uniques 859; 2024-12-03T02:34:35,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:35,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:35,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193275745, max timestamp=1733193275774Num uniques 859; 2024-12-03T02:34:35,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:35,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193335858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:35,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193335915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:35,917 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:35,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193335924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:35,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193335924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:35,929 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:35,929 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:35,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193335965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:36,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/fa1cc4540d0949c49fc1566bed07ea3c 2024-12-03T02:34:36,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/29195d21b7874808b2d8be092d7910d8 is 50, key is test_row_0/B:col10/1733193275699/Put/seqid=0 2024-12-03T02:34:36,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742098_1274 (size=12151) 2024-12-03T02:34:36,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193336174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193336484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:36,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/29195d21b7874808b2d8be092d7910d8 2024-12-03T02:34:36,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/e60901f5aca94a629b7d0b711d6ab23f is 50, key is test_row_0/C:col10/1733193275699/Put/seqid=0 2024-12-03T02:34:36,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742099_1275 (size=12151) 2024-12-03T02:34:36,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193336575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:36,579 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T02:34:36,665 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:36,665 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-03T02:34:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T02:34:36,667 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:36,667 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:36,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T02:34:36,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T02:34:36,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:36,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:36,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:36,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:36,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:36,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T02:34:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:36,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:36,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/e60901f5aca94a629b7d0b711d6ab23f 2024-12-03T02:34:36,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T02:34:36,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/fa1cc4540d0949c49fc1566bed07ea3c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fa1cc4540d0949c49fc1566bed07ea3c 2024-12-03T02:34:36,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fa1cc4540d0949c49fc1566bed07ea3c, entries=200, sequenceid=196, filesize=14.2 K 2024-12-03T02:34:36,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/29195d21b7874808b2d8be092d7910d8 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/29195d21b7874808b2d8be092d7910d8 2024-12-03T02:34:36,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/29195d21b7874808b2d8be092d7910d8, entries=150, sequenceid=196, filesize=11.9 K 2024-12-03T02:34:36,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/e60901f5aca94a629b7d0b711d6ab23f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/e60901f5aca94a629b7d0b711d6ab23f 2024-12-03T02:34:36,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:36,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193336994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:36,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/e60901f5aca94a629b7d0b711d6ab23f, entries=150, sequenceid=196, filesize=11.9 K 2024-12-03T02:34:37,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ee5b914e88815d61556ce041d081a81b in 1291ms, sequenceid=196, compaction requested=true 2024-12-03T02:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:37,000 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:37,000 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:37,001 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:37,001 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:37,001 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/A is initiating minor compaction (all files) 2024-12-03T02:34:37,001 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/B is initiating minor compaction (all files) 2024-12-03T02:34:37,002 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/A in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:37,002 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/B in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:37,002 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/12f40b2f19034ca5b2b1c78cfe2bd687, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/64142b41cda74167850bbe779ef2705e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/e33551509f2544ac8ff6e2569db1f75e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/29195d21b7874808b2d8be092d7910d8] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=47.8 K 2024-12-03T02:34:37,002 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a802042ed56e44738a697b7d4c05263d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/591cd13fac8d4a94ba54cc85f94b8045, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bbb493b710fa4c5a80cc807f8da7e1b6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fa1cc4540d0949c49fc1566bed07ea3c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=50.1 K 2024-12-03T02:34:37,002 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 12f40b2f19034ca5b2b1c78cfe2bd687, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733193271562 2024-12-03T02:34:37,002 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a802042ed56e44738a697b7d4c05263d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733193271562 2024-12-03T02:34:37,002 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 64142b41cda74167850bbe779ef2705e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733193272392 2024-12-03T02:34:37,002 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 591cd13fac8d4a94ba54cc85f94b8045, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733193272392 2024-12-03T02:34:37,003 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e33551509f2544ac8ff6e2569db1f75e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733193273757 2024-12-03T02:34:37,003 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbb493b710fa4c5a80cc807f8da7e1b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733193273757 2024-12-03T02:34:37,003 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa1cc4540d0949c49fc1566bed07ea3c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733193275694 2024-12-03T02:34:37,003 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 29195d21b7874808b2d8be092d7910d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733193275699 2024-12-03T02:34:37,011 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:37,011 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/9f86b5bc507f4d2cbb547be0b96bf8d0 is 50, key is test_row_0/A:col10/1733193275699/Put/seqid=0 2024-12-03T02:34:37,014 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:37,014 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/bb3c42a1ffe041ac91bff103f29e7ee5 is 50, key is test_row_0/B:col10/1733193275699/Put/seqid=0 2024-12-03T02:34:37,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742100_1276 (size=12629) 2024-12-03T02:34:37,022 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/9f86b5bc507f4d2cbb547be0b96bf8d0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9f86b5bc507f4d2cbb547be0b96bf8d0 2024-12-03T02:34:37,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742101_1277 (size=12629) 2024-12-03T02:34:37,028 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee5b914e88815d61556ce041d081a81b/A of ee5b914e88815d61556ce041d081a81b into 9f86b5bc507f4d2cbb547be0b96bf8d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:37,028 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:37,028 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/A, priority=12, startTime=1733193277000; duration=0sec 2024-12-03T02:34:37,028 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:37,028 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:A 2024-12-03T02:34:37,029 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:34:37,030 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:34:37,030 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/C is initiating minor compaction (all files) 2024-12-03T02:34:37,030 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/C in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:37,030 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/aaecb1bb12694509b88f66d5dd94344f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/975466afe8f443bf9c8165648c7d11cf, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1eb53a6d73ff436da1a69e7b495d58f4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/e60901f5aca94a629b7d0b711d6ab23f] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=47.8 K 2024-12-03T02:34:37,030 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaecb1bb12694509b88f66d5dd94344f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733193271562 2024-12-03T02:34:37,031 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 975466afe8f443bf9c8165648c7d11cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733193272392 2024-12-03T02:34:37,031 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eb53a6d73ff436da1a69e7b495d58f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733193273757 2024-12-03T02:34:37,031 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60901f5aca94a629b7d0b711d6ab23f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733193275699 2024-12-03T02:34:37,037 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:37,037 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/77b30071108940c494867b902f025eab is 50, key is test_row_0/C:col10/1733193275699/Put/seqid=0 2024-12-03T02:34:37,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742102_1278 (size=12629) 2024-12-03T02:34:37,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T02:34:37,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:37,125 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T02:34:37,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:37,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/c6343fa187914905ad52c068af2aead8 is 50, key is test_row_0/A:col10/1733193275834/Put/seqid=0 2024-12-03T02:34:37,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742103_1279 (size=12151) 2024-12-03T02:34:37,134 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/c6343fa187914905ad52c068af2aead8 2024-12-03T02:34:37,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/b8112e72778b4dde82dee70e6ae45f63 is 50, key is test_row_0/B:col10/1733193275834/Put/seqid=0 2024-12-03T02:34:37,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742104_1280 (size=12151) 2024-12-03T02:34:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T02:34:37,431 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/bb3c42a1ffe041ac91bff103f29e7ee5 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/bb3c42a1ffe041ac91bff103f29e7ee5 2024-12-03T02:34:37,454 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee5b914e88815d61556ce041d081a81b/B of ee5b914e88815d61556ce041d081a81b into bb3c42a1ffe041ac91bff103f29e7ee5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:37,454 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:37,454 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/B, priority=12, startTime=1733193277000; duration=0sec 2024-12-03T02:34:37,454 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:37,454 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:B 2024-12-03T02:34:37,462 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/77b30071108940c494867b902f025eab as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/77b30071108940c494867b902f025eab 2024-12-03T02:34:37,493 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ee5b914e88815d61556ce041d081a81b/C of ee5b914e88815d61556ce041d081a81b into 77b30071108940c494867b902f025eab(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:37,494 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:37,494 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/C, priority=12, startTime=1733193277000; duration=0sec 2024-12-03T02:34:37,494 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:37,494 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:C 2024-12-03T02:34:37,546 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/b8112e72778b4dde82dee70e6ae45f63 2024-12-03T02:34:37,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/5a3f05c3b73f40cda2d7e5daaf7f325e is 50, key is test_row_0/C:col10/1733193275834/Put/seqid=0 2024-12-03T02:34:37,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742105_1281 (size=12151) 2024-12-03T02:34:37,567 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/5a3f05c3b73f40cda2d7e5daaf7f325e 2024-12-03T02:34:37,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/c6343fa187914905ad52c068af2aead8 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/c6343fa187914905ad52c068af2aead8 2024-12-03T02:34:37,589 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/c6343fa187914905ad52c068af2aead8, entries=150, sequenceid=234, filesize=11.9 K 2024-12-03T02:34:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/b8112e72778b4dde82dee70e6ae45f63 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/b8112e72778b4dde82dee70e6ae45f63 2024-12-03T02:34:37,598 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/b8112e72778b4dde82dee70e6ae45f63, entries=150, sequenceid=234, filesize=11.9 K 2024-12-03T02:34:37,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/5a3f05c3b73f40cda2d7e5daaf7f325e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/5a3f05c3b73f40cda2d7e5daaf7f325e 2024-12-03T02:34:37,612 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/5a3f05c3b73f40cda2d7e5daaf7f325e, entries=150, sequenceid=234, filesize=11.9 K 2024-12-03T02:34:37,618 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for ee5b914e88815d61556ce041d081a81b in 493ms, sequenceid=234, compaction requested=false 2024-12-03T02:34:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:37,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-03T02:34:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-03T02:34:37,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-03T02:34:37,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 954 msec 2024-12-03T02:34:37,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 960 msec 2024-12-03T02:34:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T02:34:37,805 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:37,808 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-03T02:34:37,811 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:37,814 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:37,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T02:34:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T02:34:37,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:34:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:37,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:37,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:37,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:37,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:37,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:37,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:37,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/fc52f55a1e2549b3abcf42c337c99ecf is 50, key is test_row_0/A:col10/1733193277963/Put/seqid=0 2024-12-03T02:34:38,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193338016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193338019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193278006, max timestamp=1733193278023Num uniques 859; 2024-12-03T02:34:38,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:38,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193278006, max timestamp=1733193278023Num uniques 859; 2024-12-03T02:34:38,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:38,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193278011, max timestamp=1733193278023Num uniques 859; 2024-12-03T02:34:38,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:34:38,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742106_1282 (size=12151) 2024-12-03T02:34:38,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193338101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193338102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:38,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:38,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T02:34:38,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193338135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193338135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193338215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193338215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:38,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:38,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193338344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193338345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193338427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193338435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:38,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:38,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T02:34:38,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/fc52f55a1e2549b3abcf42c337c99ecf 2024-12-03T02:34:38,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a61b99b0fa644ffeb456320d3ff984ca is 50, key is test_row_0/B:col10/1733193277963/Put/seqid=0 2024-12-03T02:34:38,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742107_1283 (size=12151) 2024-12-03T02:34:38,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:38,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:38,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193338655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193338655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193338735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193338747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:38,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:38,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:38,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a61b99b0fa644ffeb456320d3ff984ca 2024-12-03T02:34:38,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:38,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:38,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/2e80ad22f10f483e8c1c933636d4b22c is 50, key is test_row_0/C:col10/1733193277963/Put/seqid=0 2024-12-03T02:34:38,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742108_1284 (size=12151) 2024-12-03T02:34:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T02:34:39,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:39,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193339166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:39,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193339176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:39,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:39,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:39,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:39,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:39,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193339254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:39,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193339265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:39,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/2e80ad22f10f483e8c1c933636d4b22c 2024-12-03T02:34:39,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/fc52f55a1e2549b3abcf42c337c99ecf as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fc52f55a1e2549b3abcf42c337c99ecf 2024-12-03T02:34:39,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fc52f55a1e2549b3abcf42c337c99ecf, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T02:34:39,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a61b99b0fa644ffeb456320d3ff984ca as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a61b99b0fa644ffeb456320d3ff984ca 2024-12-03T02:34:39,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a61b99b0fa644ffeb456320d3ff984ca, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T02:34:39,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/2e80ad22f10f483e8c1c933636d4b22c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2e80ad22f10f483e8c1c933636d4b22c 2024-12-03T02:34:39,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2e80ad22f10f483e8c1c933636d4b22c, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T02:34:39,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:39,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:39,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:39,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:39,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.13 KB/120965 for ee5b914e88815d61556ce041d081a81b in 1429ms, sequenceid=248, compaction requested=true 2024-12-03T02:34:39,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:39,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:39,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:39,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:39,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:34:39,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:39,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T02:34:39,393 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:39,393 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:39,395 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:39,396 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/A is initiating minor compaction (all files) 2024-12-03T02:34:39,396 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/A in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,396 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9f86b5bc507f4d2cbb547be0b96bf8d0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/c6343fa187914905ad52c068af2aead8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fc52f55a1e2549b3abcf42c337c99ecf] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.1 K 2024-12-03T02:34:39,396 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f86b5bc507f4d2cbb547be0b96bf8d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733193275699 2024-12-03T02:34:39,397 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6343fa187914905ad52c068af2aead8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733193275834 2024-12-03T02:34:39,398 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc52f55a1e2549b3abcf42c337c99ecf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733193277954 2024-12-03T02:34:39,398 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:39,398 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/C is initiating minor compaction (all files) 2024-12-03T02:34:39,398 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/C in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,398 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/77b30071108940c494867b902f025eab, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/5a3f05c3b73f40cda2d7e5daaf7f325e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2e80ad22f10f483e8c1c933636d4b22c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.1 K 2024-12-03T02:34:39,399 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 77b30071108940c494867b902f025eab, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733193275699 2024-12-03T02:34:39,404 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a3f05c3b73f40cda2d7e5daaf7f325e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733193275834 2024-12-03T02:34:39,405 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e80ad22f10f483e8c1c933636d4b22c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733193277954 2024-12-03T02:34:39,449 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:39,449 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/2d6e3624fb2b4dc98202f4a547b6274a is 50, key is test_row_0/A:col10/1733193277963/Put/seqid=0 2024-12-03T02:34:39,477 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#C#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:39,478 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/886d024c9db447a89d85640e867888b8 is 50, key is test_row_0/C:col10/1733193277963/Put/seqid=0 2024-12-03T02:34:39,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742109_1285 (size=12731) 2024-12-03T02:34:39,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742110_1286 (size=12731) 2024-12-03T02:34:39,540 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/2d6e3624fb2b4dc98202f4a547b6274a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2d6e3624fb2b4dc98202f4a547b6274a 2024-12-03T02:34:39,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T02:34:39,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,555 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=118.13 KB heapSize=334.46 KB 2024-12-03T02:34:39,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:39,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:39,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:39,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:39,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:39,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:39,557 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/886d024c9db447a89d85640e867888b8 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/886d024c9db447a89d85640e867888b8 2024-12-03T02:34:39,563 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/A of ee5b914e88815d61556ce041d081a81b into 2d6e3624fb2b4dc98202f4a547b6274a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:39,564 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:39,564 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/A, priority=13, startTime=1733193279392; duration=0sec 2024-12-03T02:34:39,564 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:39,564 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:A 2024-12-03T02:34:39,564 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:39,576 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:39,576 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/B is initiating minor compaction (all files) 2024-12-03T02:34:39,576 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/B in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:39,576 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/bb3c42a1ffe041ac91bff103f29e7ee5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/b8112e72778b4dde82dee70e6ae45f63, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a61b99b0fa644ffeb456320d3ff984ca] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.1 K 2024-12-03T02:34:39,580 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb3c42a1ffe041ac91bff103f29e7ee5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733193275699 2024-12-03T02:34:39,588 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8112e72778b4dde82dee70e6ae45f63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733193275834 2024-12-03T02:34:39,588 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a61b99b0fa644ffeb456320d3ff984ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733193277954 2024-12-03T02:34:39,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/972ddf058b954abdb6445602a0b18077 is 50, key is test_row_0/A:col10/1733193278100/Put/seqid=0 2024-12-03T02:34:39,598 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/C of ee5b914e88815d61556ce041d081a81b into 886d024c9db447a89d85640e867888b8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:39,598 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:39,598 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/C, priority=13, startTime=1733193279392; duration=0sec 2024-12-03T02:34:39,598 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:39,598 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:C 2024-12-03T02:34:39,609 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:39,610 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/df6973bca1f647aa89fde29874fe6402 is 50, key is test_row_0/B:col10/1733193277963/Put/seqid=0 2024-12-03T02:34:39,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742111_1287 (size=12301) 2024-12-03T02:34:39,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742112_1288 (size=12731) 2024-12-03T02:34:39,707 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/df6973bca1f647aa89fde29874fe6402 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/df6973bca1f647aa89fde29874fe6402 2024-12-03T02:34:39,720 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/B of ee5b914e88815d61556ce041d081a81b into df6973bca1f647aa89fde29874fe6402(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:39,720 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:39,720 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/B, priority=13, startTime=1733193279392; duration=0sec 2024-12-03T02:34:39,724 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:39,724 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:B 2024-12-03T02:34:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T02:34:40,045 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/972ddf058b954abdb6445602a0b18077 2024-12-03T02:34:40,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/ee972ebfcb6e454c9f950c75fec6ebfe is 50, key is test_row_0/B:col10/1733193278100/Put/seqid=0 2024-12-03T02:34:40,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742113_1289 (size=12301) 2024-12-03T02:34:40,063 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/ee972ebfcb6e454c9f950c75fec6ebfe 2024-12-03T02:34:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/380b446827c54273976e1e8976d9f131 is 50, key is test_row_0/C:col10/1733193278100/Put/seqid=0 2024-12-03T02:34:40,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742114_1290 (size=12301) 2024-12-03T02:34:40,122 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/380b446827c54273976e1e8976d9f131 2024-12-03T02:34:40,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/972ddf058b954abdb6445602a0b18077 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/972ddf058b954abdb6445602a0b18077 2024-12-03T02:34:40,154 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/972ddf058b954abdb6445602a0b18077, entries=150, sequenceid=282, filesize=12.0 K 2024-12-03T02:34:40,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/ee972ebfcb6e454c9f950c75fec6ebfe as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ee972ebfcb6e454c9f950c75fec6ebfe 2024-12-03T02:34:40,162 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ee972ebfcb6e454c9f950c75fec6ebfe, entries=150, sequenceid=282, filesize=12.0 K 2024-12-03T02:34:40,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/380b446827c54273976e1e8976d9f131 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/380b446827c54273976e1e8976d9f131 2024-12-03T02:34:40,169 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/380b446827c54273976e1e8976d9f131, entries=150, sequenceid=282, filesize=12.0 K 2024-12-03T02:34:40,171 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~118.13 KB/120965, heapSize ~334.41 KB/342440, currentSize=0 B/0 for ee5b914e88815d61556ce041d081a81b in 615ms, sequenceid=282, compaction requested=false 2024-12-03T02:34:40,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:40,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:40,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-03T02:34:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-03T02:34:40,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-03T02:34:40,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3580 sec 2024-12-03T02:34:40,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3660 sec 2024-12-03T02:34:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:40,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:34:40,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:40,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:40,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:40,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:40,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:40,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:40,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/70aa6321ee3449719f1257514aeedb08 is 50, key is test_row_0/A:col10/1733193280192/Put/seqid=0 2024-12-03T02:34:40,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742115_1291 (size=12301) 2024-12-03T02:34:40,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193280233, max timestamp=1733193280250Num uniques 859; 2024-12-03T02:34:40,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:34:40,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193280239, max timestamp=1733193280250Num uniques 859; 2024-12-03T02:34:40,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:40,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193280239, max timestamp=1733193280250Num uniques 859; 2024-12-03T02:34:40,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:34:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193280278, max timestamp=1733193280297Num uniques 1018; 2024-12-03T02:34:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9970, totalHeapSize=22912, min timestamp=1733193280278, max timestamp=1733193280297Num uniques 1068; 2024-12-03T02:34:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T02:34:40,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48900, new segment data size=9970, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-03T02:34:40,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193280278, max timestamp=1733193280297Num uniques 1018; 2024-12-03T02:34:40,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T02:34:40,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193340329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193340329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193340336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193340338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193340445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193340445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193340445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193340445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57776 deadline: 1733193340594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,597 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in ee5b914e88815d61556ce041d081a81b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:34:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/70aa6321ee3449719f1257514aeedb08 2024-12-03T02:34:40,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193340655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193340655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193340655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193340656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/ef707bd583d6472bad6a4ca950a44fb9 is 50, key is test_row_0/B:col10/1733193280192/Put/seqid=0 2024-12-03T02:34:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742116_1292 (size=12301) 2024-12-03T02:34:40,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/ef707bd583d6472bad6a4ca950a44fb9 2024-12-03T02:34:40,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/d5cda8ba25284251aa74934ca436eee2 is 50, key is test_row_0/C:col10/1733193280192/Put/seqid=0 2024-12-03T02:34:40,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742117_1293 (size=12301) 2024-12-03T02:34:40,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/d5cda8ba25284251aa74934ca436eee2 2024-12-03T02:34:40,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/70aa6321ee3449719f1257514aeedb08 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/70aa6321ee3449719f1257514aeedb08 2024-12-03T02:34:40,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/70aa6321ee3449719f1257514aeedb08, entries=150, sequenceid=295, filesize=12.0 K 2024-12-03T02:34:40,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/ef707bd583d6472bad6a4ca950a44fb9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ef707bd583d6472bad6a4ca950a44fb9 2024-12-03T02:34:40,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ef707bd583d6472bad6a4ca950a44fb9, entries=150, sequenceid=295, filesize=12.0 K 2024-12-03T02:34:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/d5cda8ba25284251aa74934ca436eee2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/d5cda8ba25284251aa74934ca436eee2 2024-12-03T02:34:40,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/d5cda8ba25284251aa74934ca436eee2, entries=150, sequenceid=295, filesize=12.0 K 2024-12-03T02:34:40,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for ee5b914e88815d61556ce041d081a81b in 682ms, sequenceid=295, compaction requested=true 2024-12-03T02:34:40,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:40,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:40,876 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:40,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:40,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:40,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:40,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:40,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:34:40,876 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:40,879 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:40,879 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/A is initiating minor compaction (all files) 2024-12-03T02:34:40,879 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/A in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:40,879 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2d6e3624fb2b4dc98202f4a547b6274a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/972ddf058b954abdb6445602a0b18077, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/70aa6321ee3449719f1257514aeedb08] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.5 K 2024-12-03T02:34:40,880 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:40,880 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/B is initiating minor compaction (all files) 2024-12-03T02:34:40,881 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/B in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:40,881 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/df6973bca1f647aa89fde29874fe6402, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ee972ebfcb6e454c9f950c75fec6ebfe, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ef707bd583d6472bad6a4ca950a44fb9] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.5 K 2024-12-03T02:34:40,891 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting df6973bca1f647aa89fde29874fe6402, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733193277954 2024-12-03T02:34:40,891 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6e3624fb2b4dc98202f4a547b6274a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733193277954 2024-12-03T02:34:40,893 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 972ddf058b954abdb6445602a0b18077, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733193278079 2024-12-03T02:34:40,894 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70aa6321ee3449719f1257514aeedb08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733193280185 2024-12-03T02:34:40,896 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ee972ebfcb6e454c9f950c75fec6ebfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733193278079 2024-12-03T02:34:40,897 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ef707bd583d6472bad6a4ca950a44fb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733193280185 2024-12-03T02:34:40,909 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:40,910 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/6c86b837fbb04d9d86ee7a20bdf80097 is 50, key is test_row_0/A:col10/1733193280192/Put/seqid=0 2024-12-03T02:34:40,913 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:40,913 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/abb041c9dfc74e84bd121abe1e3e12a9 is 50, key is test_row_0/B:col10/1733193280192/Put/seqid=0 2024-12-03T02:34:40,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742119_1295 (size=12983) 2024-12-03T02:34:40,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742118_1294 (size=12983) 2024-12-03T02:34:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:40,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=145.36 KB heapSize=414.92 KB 2024-12-03T02:34:40,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:40,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:34:40,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:40,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:34:40,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:40,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:34:40,972 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/6c86b837fbb04d9d86ee7a20bdf80097 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/6c86b837fbb04d9d86ee7a20bdf80097 2024-12-03T02:34:40,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/34c44d6832494149a0e968b53235e004 is 50, key is test_row_0/A:col10/1733193280968/Put/seqid=0 2024-12-03T02:34:40,981 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/A of ee5b914e88815d61556ce041d081a81b into 6c86b837fbb04d9d86ee7a20bdf80097(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:40,981 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:40,981 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/A, priority=13, startTime=1733193280875; duration=0sec 2024-12-03T02:34:40,981 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:40,981 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:A 2024-12-03T02:34:40,981 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:40,982 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:40,983 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/C is initiating minor compaction (all files) 2024-12-03T02:34:40,983 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/C in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:40,983 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/886d024c9db447a89d85640e867888b8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/380b446827c54273976e1e8976d9f131, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/d5cda8ba25284251aa74934ca436eee2] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.5 K 2024-12-03T02:34:40,985 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 886d024c9db447a89d85640e867888b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733193277954 2024-12-03T02:34:40,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193340982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,989 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 380b446827c54273976e1e8976d9f131, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733193278079 2024-12-03T02:34:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,990 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5cda8ba25284251aa74934ca436eee2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733193280185 2024-12-03T02:34:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193340982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193340983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193340984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:40,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:40,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742120_1296 (size=12301) 2024-12-03T02:34:40,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/34c44d6832494149a0e968b53235e004 2024-12-03T02:34:41,005 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:41,005 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/bef548fd92af49e4bcb83bb258d39f56 is 50, key is test_row_0/C:col10/1733193280192/Put/seqid=0 2024-12-03T02:34:41,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a5b511d5c2e547369d36d11bcdebfc7d is 50, key is test_row_0/B:col10/1733193280968/Put/seqid=0 2024-12-03T02:34:41,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742121_1297 (size=12983) 2024-12-03T02:34:41,042 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/bef548fd92af49e4bcb83bb258d39f56 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/bef548fd92af49e4bcb83bb258d39f56 2024-12-03T02:34:41,048 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/C of ee5b914e88815d61556ce041d081a81b into bef548fd92af49e4bcb83bb258d39f56(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:41,048 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:41,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742122_1298 (size=12301) 2024-12-03T02:34:41,048 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/C, priority=13, startTime=1733193280876; duration=0sec 2024-12-03T02:34:41,049 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:41,049 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:C 2024-12-03T02:34:41,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a5b511d5c2e547369d36d11bcdebfc7d 2024-12-03T02:34:41,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/eeb6e41c01fc4702bf69011b2fa57b82 is 50, key is test_row_0/C:col10/1733193280968/Put/seqid=0 2024-12-03T02:34:41,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742123_1299 (size=12301) 2024-12-03T02:34:41,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193341094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193341095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193341095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193341095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193341305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193341305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193341305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193341305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,365 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/abb041c9dfc74e84bd121abe1e3e12a9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/abb041c9dfc74e84bd121abe1e3e12a9 2024-12-03T02:34:41,369 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/B of ee5b914e88815d61556ce041d081a81b into abb041c9dfc74e84bd121abe1e3e12a9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:41,369 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:41,369 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/B, priority=13, startTime=1733193280876; duration=0sec 2024-12-03T02:34:41,369 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:41,369 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:B 2024-12-03T02:34:41,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/eeb6e41c01fc4702bf69011b2fa57b82 2024-12-03T02:34:41,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/34c44d6832494149a0e968b53235e004 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/34c44d6832494149a0e968b53235e004 2024-12-03T02:34:41,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/34c44d6832494149a0e968b53235e004, entries=150, sequenceid=351, filesize=12.0 K 2024-12-03T02:34:41,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/a5b511d5c2e547369d36d11bcdebfc7d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a5b511d5c2e547369d36d11bcdebfc7d 2024-12-03T02:34:41,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a5b511d5c2e547369d36d11bcdebfc7d, entries=150, sequenceid=351, filesize=12.0 K 2024-12-03T02:34:41,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/eeb6e41c01fc4702bf69011b2fa57b82 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/eeb6e41c01fc4702bf69011b2fa57b82 2024-12-03T02:34:41,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/eeb6e41c01fc4702bf69011b2fa57b82, entries=150, sequenceid=351, filesize=12.0 K 2024-12-03T02:34:41,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~434.80 KB/445232, currentSize=33.54 KB/34350 for ee5b914e88815d61556ce041d081a81b in 562ms, sequenceid=351, compaction requested=false 2024-12-03T02:34:41,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:41,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:34:41,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:41,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/a4d889833ace496491a44bf2b9ef3c58 is 50, key is test_row_0/A:col10/1733193281618/Put/seqid=0 2024-12-03T02:34:41,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193341658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193341658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742124_1300 (size=12301) 2024-12-03T02:34:41,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193341664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193341668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193341775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193341775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193341775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193341784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193341985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193341987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193341987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T02:34:42,001 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:42,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193341999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,007 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-03T02:34:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T02:34:42,023 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:42,031 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:42,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:42,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/a4d889833ace496491a44bf2b9ef3c58 2024-12-03T02:34:42,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/702ac4ac6d99455bbc1b99c35255b485 is 50, key is test_row_0/B:col10/1733193281618/Put/seqid=0 2024-12-03T02:34:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T02:34:42,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742125_1301 (size=12301) 2024-12-03T02:34:42,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/702ac4ac6d99455bbc1b99c35255b485 2024-12-03T02:34:42,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T02:34:42,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:42,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/7ea526ece1014b50b1d76ebe22d86572 is 50, key is test_row_0/C:col10/1733193281618/Put/seqid=0 2024-12-03T02:34:42,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742126_1302 (size=12301) 2024-12-03T02:34:42,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193342294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193342305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193342306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193342315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T02:34:42,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T02:34:42,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:42,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T02:34:42,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:42,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T02:34:42,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T02:34:42,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:42,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:42,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/7ea526ece1014b50b1d76ebe22d86572 2024-12-03T02:34:42,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/a4d889833ace496491a44bf2b9ef3c58 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a4d889833ace496491a44bf2b9ef3c58 2024-12-03T02:34:42,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a4d889833ace496491a44bf2b9ef3c58, entries=150, sequenceid=366, filesize=12.0 K 2024-12-03T02:34:42,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/702ac4ac6d99455bbc1b99c35255b485 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/702ac4ac6d99455bbc1b99c35255b485 2024-12-03T02:34:42,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/702ac4ac6d99455bbc1b99c35255b485, entries=150, sequenceid=366, filesize=12.0 K 2024-12-03T02:34:42,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/7ea526ece1014b50b1d76ebe22d86572 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7ea526ece1014b50b1d76ebe22d86572 2024-12-03T02:34:42,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7ea526ece1014b50b1d76ebe22d86572, entries=150, sequenceid=366, filesize=12.0 K 2024-12-03T02:34:42,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for ee5b914e88815d61556ce041d081a81b in 1184ms, sequenceid=366, compaction requested=true 2024-12-03T02:34:42,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:42,804 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:42,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:42,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:42,805 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:42,805 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:42,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:42,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:42,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:42,805 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/A is initiating minor compaction (all files) 2024-12-03T02:34:42,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:42,805 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/A in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,805 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/6c86b837fbb04d9d86ee7a20bdf80097, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/34c44d6832494149a0e968b53235e004, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a4d889833ace496491a44bf2b9ef3c58] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.7 K 2024-12-03T02:34:42,806 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c86b837fbb04d9d86ee7a20bdf80097, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733193280185 2024-12-03T02:34:42,811 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c44d6832494149a0e968b53235e004, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733193280332 2024-12-03T02:34:42,814 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4d889833ace496491a44bf2b9ef3c58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733193281618 2024-12-03T02:34:42,815 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:42,815 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/B is initiating minor compaction (all files) 2024-12-03T02:34:42,815 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/B in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,815 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/abb041c9dfc74e84bd121abe1e3e12a9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a5b511d5c2e547369d36d11bcdebfc7d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/702ac4ac6d99455bbc1b99c35255b485] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.7 K 2024-12-03T02:34:42,818 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting abb041c9dfc74e84bd121abe1e3e12a9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733193280185 2024-12-03T02:34:42,819 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b511d5c2e547369d36d11bcdebfc7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733193280332 2024-12-03T02:34:42,821 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 702ac4ac6d99455bbc1b99c35255b485, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733193281618 2024-12-03T02:34:42,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T02:34:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,824 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T02:34:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:42,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:34:42,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. as already flushing 2024-12-03T02:34:42,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/37decca0c05f47bc9db0c527621b0256 is 50, key is test_row_0/A:col10/1733193281661/Put/seqid=0 2024-12-03T02:34:42,845 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:42,846 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/f9cf9ebc14d8466a8341b5dba2e997aa is 50, key is test_row_0/A:col10/1733193281618/Put/seqid=0 2024-12-03T02:34:42,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193342842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193342846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193342848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193342849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,857 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#B#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:42,858 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/d6bf8e4acd1348699bbe8499c35db71f is 50, key is test_row_0/B:col10/1733193281618/Put/seqid=0 2024-12-03T02:34:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742128_1304 (size=14741) 2024-12-03T02:34:42,884 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/37decca0c05f47bc9db0c527621b0256 2024-12-03T02:34:42,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742127_1303 (size=13085) 2024-12-03T02:34:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/9442425b79f943ba9c9e9f35ef708b48 is 50, key is test_row_0/B:col10/1733193281661/Put/seqid=0 2024-12-03T02:34:42,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742129_1305 (size=13085) 2024-12-03T02:34:42,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193342956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742130_1306 (size=12301) 2024-12-03T02:34:42,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,971 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/d6bf8e4acd1348699bbe8499c35db71f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/d6bf8e4acd1348699bbe8499c35db71f 2024-12-03T02:34:42,973 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/9442425b79f943ba9c9e9f35ef708b48 2024-12-03T02:34:42,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193342971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193342971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193342971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:42,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:42,986 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/B of ee5b914e88815d61556ce041d081a81b into d6bf8e4acd1348699bbe8499c35db71f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:42,986 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:42,986 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/B, priority=13, startTime=1733193282805; duration=0sec 2024-12-03T02:34:42,986 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:42,987 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:B 2024-12-03T02:34:42,987 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/8bc1fde728b04b87bd61af3bda293d12 is 50, key is test_row_0/C:col10/1733193281661/Put/seqid=0 2024-12-03T02:34:42,990 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:42,990 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/C is initiating minor compaction (all files) 2024-12-03T02:34:42,990 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/C in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:42,990 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/bef548fd92af49e4bcb83bb258d39f56, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/eeb6e41c01fc4702bf69011b2fa57b82, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7ea526ece1014b50b1d76ebe22d86572] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.7 K 2024-12-03T02:34:42,991 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting bef548fd92af49e4bcb83bb258d39f56, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733193280185 2024-12-03T02:34:42,991 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting eeb6e41c01fc4702bf69011b2fa57b82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733193280332 2024-12-03T02:34:42,995 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ea526ece1014b50b1d76ebe22d86572, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733193281618 2024-12-03T02:34:43,036 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:43,036 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/2d5add15cb2b48cbbe828412b3fdc052 is 50, key is test_row_0/C:col10/1733193281618/Put/seqid=0 2024-12-03T02:34:43,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742132_1308 (size=13085) 2024-12-03T02:34:43,072 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/2d5add15cb2b48cbbe828412b3fdc052 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2d5add15cb2b48cbbe828412b3fdc052 2024-12-03T02:34:43,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742131_1307 (size=12301) 2024-12-03T02:34:43,082 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/C of ee5b914e88815d61556ce041d081a81b into 2d5add15cb2b48cbbe828412b3fdc052(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:43,082 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:43,082 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/C, priority=13, startTime=1733193282805; duration=0sec 2024-12-03T02:34:43,082 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:43,082 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:C 2024-12-03T02:34:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193343175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T02:34:43,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193343189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:43,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193343189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:43,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193343189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:43,348 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/f9cf9ebc14d8466a8341b5dba2e997aa as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f9cf9ebc14d8466a8341b5dba2e997aa 2024-12-03T02:34:43,354 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/A of ee5b914e88815d61556ce041d081a81b into f9cf9ebc14d8466a8341b5dba2e997aa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:43,354 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:43,354 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/A, priority=13, startTime=1733193282804; duration=0sec 2024-12-03T02:34:43,354 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:43,354 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:A 2024-12-03T02:34:43,480 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/8bc1fde728b04b87bd61af3bda293d12 2024-12-03T02:34:43,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/37decca0c05f47bc9db0c527621b0256 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/37decca0c05f47bc9db0c527621b0256 2024-12-03T02:34:43,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57810 deadline: 1733193343485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:43,494 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/37decca0c05f47bc9db0c527621b0256, entries=200, sequenceid=388, filesize=14.4 K 2024-12-03T02:34:43,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/9442425b79f943ba9c9e9f35ef708b48 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9442425b79f943ba9c9e9f35ef708b48 2024-12-03T02:34:43,513 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9442425b79f943ba9c9e9f35ef708b48, entries=150, sequenceid=388, filesize=12.0 K 2024-12-03T02:34:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57794 deadline: 1733193343505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57748 deadline: 1733193343505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57764 deadline: 1733193343506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/8bc1fde728b04b87bd61af3bda293d12 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/8bc1fde728b04b87bd61af3bda293d12 2024-12-03T02:34:43,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:34:43,536 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/8bc1fde728b04b87bd61af3bda293d12, entries=150, sequenceid=388, filesize=12.0 K 2024-12-03T02:34:43,538 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for ee5b914e88815d61556ce041d081a81b in 715ms, sequenceid=388, compaction requested=false 2024-12-03T02:34:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-03T02:34:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-03T02:34:43,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-03T02:34:43,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2024-12-03T02:34:43,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5350 sec 2024-12-03T02:34:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:44,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T02:34:44,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:44,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:44,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:44,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:44,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:44,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:44,021 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-12-03T02:34:44,021 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:44,021 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,021 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:44,024 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-12-03T02:34:44,024 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:44,025 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:44,025 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,026 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-12-03T02:34:44,026 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:44,026 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,026 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:44,027 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-12-03T02:34:44,027 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:44,027 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,027 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-12-03T02:34:44,027 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:44,027 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,027 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:44,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/f080a032c00040b19ccfcda069a5b9c4 is 50, key is test_row_0/A:col10/1733193282842/Put/seqid=0 2024-12-03T02:34:44,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:44,032 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,032 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-12-03T02:34:44,032 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:44,032 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,032 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,033 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-12-03T02:34:44,033 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:44,033 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,033 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-12-03T02:34:44,033 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:44,033 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:44,033 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:44,034 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:44,036 INFO [Thread-1094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1094. 2024-12-03T02:34:44,036 DEBUG [Thread-1094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:44,036 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,036 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:44,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:44,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742133_1309 (size=12301) 2024-12-03T02:34:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T02:34:44,195 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:44,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/f080a032c00040b19ccfcda069a5b9c4 2024-12-03T02:34:44,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/21c258e4fd494cd2be4cc59aefae571e is 50, key is test_row_0/B:col10/1733193282842/Put/seqid=0 2024-12-03T02:34:44,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742134_1310 (size=12301) 2024-12-03T02:34:44,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/21c258e4fd494cd2be4cc59aefae571e 2024-12-03T02:34:44,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/2fcecab33d72489e8a5bd65bd2079492 is 50, key is test_row_0/C:col10/1733193282842/Put/seqid=0 2024-12-03T02:34:44,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742135_1311 (size=12301) 2024-12-03T02:34:44,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/2fcecab33d72489e8a5bd65bd2079492 2024-12-03T02:34:44,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/f080a032c00040b19ccfcda069a5b9c4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f080a032c00040b19ccfcda069a5b9c4 2024-12-03T02:34:44,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f080a032c00040b19ccfcda069a5b9c4, entries=150, sequenceid=402, filesize=12.0 K 2024-12-03T02:34:44,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/21c258e4fd494cd2be4cc59aefae571e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21c258e4fd494cd2be4cc59aefae571e 2024-12-03T02:34:44,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21c258e4fd494cd2be4cc59aefae571e, entries=150, sequenceid=402, filesize=12.0 K 2024-12-03T02:34:44,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/2fcecab33d72489e8a5bd65bd2079492 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2fcecab33d72489e8a5bd65bd2079492 2024-12-03T02:34:44,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2fcecab33d72489e8a5bd65bd2079492, entries=150, sequenceid=402, filesize=12.0 K 2024-12-03T02:34:44,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=26.84 KB/27480 for ee5b914e88815d61556ce041d081a81b in 882ms, sequenceid=402, compaction requested=true 2024-12-03T02:34:44,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:44,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:34:44,898 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:44,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:44,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:34:44,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:44,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ee5b914e88815d61556ce041d081a81b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:34:44,898 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:44,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:44,899 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:44,899 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:44,899 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/A is initiating minor compaction (all files) 2024-12-03T02:34:44,899 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/B is initiating minor compaction (all files) 2024-12-03T02:34:44,899 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/A in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:44,900 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/B in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:44,900 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/d6bf8e4acd1348699bbe8499c35db71f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9442425b79f943ba9c9e9f35ef708b48, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21c258e4fd494cd2be4cc59aefae571e] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.8 K 2024-12-03T02:34:44,900 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f9cf9ebc14d8466a8341b5dba2e997aa, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/37decca0c05f47bc9db0c527621b0256, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f080a032c00040b19ccfcda069a5b9c4] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=39.2 K 2024-12-03T02:34:44,900 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d6bf8e4acd1348699bbe8499c35db71f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733193281618 2024-12-03T02:34:44,900 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9cf9ebc14d8466a8341b5dba2e997aa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733193281618 2024-12-03T02:34:44,900 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9442425b79f943ba9c9e9f35ef708b48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733193281653 2024-12-03T02:34:44,900 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37decca0c05f47bc9db0c527621b0256, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733193281653 2024-12-03T02:34:44,900 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c258e4fd494cd2be4cc59aefae571e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733193282842 2024-12-03T02:34:44,900 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting f080a032c00040b19ccfcda069a5b9c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733193282842 2024-12-03T02:34:44,917 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:44,918 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/047809ec094941c189b6c5f793de7bd6 is 50, key is test_row_0/A:col10/1733193282842/Put/seqid=0 2024-12-03T02:34:44,922 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:44,923 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/38e8aac1c55f4858af87f3293007c8b3 is 50, key is test_row_0/B:col10/1733193282842/Put/seqid=0 2024-12-03T02:34:44,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742136_1312 (size=13187) 2024-12-03T02:34:44,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742137_1313 (size=13187) 2024-12-03T02:34:45,333 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/047809ec094941c189b6c5f793de7bd6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/047809ec094941c189b6c5f793de7bd6 2024-12-03T02:34:45,338 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/A of ee5b914e88815d61556ce041d081a81b into 047809ec094941c189b6c5f793de7bd6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:45,338 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:45,338 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/A, priority=13, startTime=1733193284898; duration=0sec 2024-12-03T02:34:45,338 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:34:45,338 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:A 2024-12-03T02:34:45,338 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:34:45,340 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:34:45,340 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): ee5b914e88815d61556ce041d081a81b/C is initiating minor compaction (all files) 2024-12-03T02:34:45,340 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ee5b914e88815d61556ce041d081a81b/C in TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:45,340 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2d5add15cb2b48cbbe828412b3fdc052, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/8bc1fde728b04b87bd61af3bda293d12, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2fcecab33d72489e8a5bd65bd2079492] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp, totalSize=36.8 K 2024-12-03T02:34:45,340 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d5add15cb2b48cbbe828412b3fdc052, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733193281618 2024-12-03T02:34:45,341 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc1fde728b04b87bd61af3bda293d12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733193281653 2024-12-03T02:34:45,341 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fcecab33d72489e8a5bd65bd2079492, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733193282842 2024-12-03T02:34:45,351 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ee5b914e88815d61556ce041d081a81b#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:34:45,351 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/6d1225c3d49d4c49abc00eadbe40e14b is 50, key is test_row_0/C:col10/1733193282842/Put/seqid=0 2024-12-03T02:34:45,360 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/38e8aac1c55f4858af87f3293007c8b3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/38e8aac1c55f4858af87f3293007c8b3 2024-12-03T02:34:45,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742138_1314 (size=13187) 2024-12-03T02:34:45,371 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/B of ee5b914e88815d61556ce041d081a81b into 38e8aac1c55f4858af87f3293007c8b3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:45,371 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:45,371 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/B, priority=13, startTime=1733193284898; duration=0sec 2024-12-03T02:34:45,372 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:45,372 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:B 2024-12-03T02:34:45,381 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/6d1225c3d49d4c49abc00eadbe40e14b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/6d1225c3d49d4c49abc00eadbe40e14b 2024-12-03T02:34:45,396 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ee5b914e88815d61556ce041d081a81b/C of ee5b914e88815d61556ce041d081a81b into 6d1225c3d49d4c49abc00eadbe40e14b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:34:45,396 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ee5b914e88815d61556ce041d081a81b: 2024-12-03T02:34:45,396 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., storeName=ee5b914e88815d61556ce041d081a81b/C, priority=13, startTime=1733193284898; duration=0sec 2024-12-03T02:34:45,396 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:34:45,396 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ee5b914e88815d61556ce041d081a81b:C 2024-12-03T02:34:50,004 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T02:34:50,616 INFO [Thread-1096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1096. 2024-12-03T02:34:50,616 DEBUG [Thread-1096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:34:50,616 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:50,617 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:50,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3667 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3418 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3419 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3635 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3415 2024-12-03T02:34:50,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T02:34:50,617 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:34:50,617 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:34:50,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:50,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:34:50,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:50,618 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T02:34:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T02:34:50,621 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193290621"}]},"ts":"1733193290621"} 2024-12-03T02:34:50,622 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T02:34:50,623 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T02:34:50,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T02:34:50,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b, UNASSIGN}] 2024-12-03T02:34:50,625 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b, UNASSIGN 2024-12-03T02:34:50,626 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=ee5b914e88815d61556ce041d081a81b, regionState=CLOSING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:50,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b, UNASSIGN because future has completed 2024-12-03T02:34:50,628 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:34:50,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:34:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T02:34:50,782 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:50,782 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:34:50,782 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing ee5b914e88815d61556ce041d081a81b, disabling compactions & flushes 2024-12-03T02:34:50,782 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:50,782 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:50,782 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. after waiting 0 ms 2024-12-03T02:34:50,782 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:50,783 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(2902): Flushing ee5b914e88815d61556ce041d081a81b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T02:34:50,783 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=A 2024-12-03T02:34:50,783 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:50,783 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=B 2024-12-03T02:34:50,783 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:50,783 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ee5b914e88815d61556ce041d081a81b, store=C 2024-12-03T02:34:50,783 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:50,788 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/e5d7cb13bb574edf98a752055f3d813c is 50, key is test_row_0/A:col10/1733193284026/Put/seqid=0 2024-12-03T02:34:50,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742139_1315 (size=12301) 2024-12-03T02:34:50,804 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/e5d7cb13bb574edf98a752055f3d813c 2024-12-03T02:34:50,813 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/faa50f8a627c44e4ac45e90bdc3c7932 is 50, key is test_row_0/B:col10/1733193284026/Put/seqid=0 2024-12-03T02:34:50,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742140_1316 (size=12301) 2024-12-03T02:34:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T02:34:51,229 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/faa50f8a627c44e4ac45e90bdc3c7932 2024-12-03T02:34:51,235 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/a73c3950ac5046f0a445db679c6c318b is 50, key is test_row_0/C:col10/1733193284026/Put/seqid=0 2024-12-03T02:34:51,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742141_1317 (size=12301) 2024-12-03T02:34:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T02:34:51,640 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/a73c3950ac5046f0a445db679c6c318b 2024-12-03T02:34:51,646 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/A/e5d7cb13bb574edf98a752055f3d813c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e5d7cb13bb574edf98a752055f3d813c 2024-12-03T02:34:51,650 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e5d7cb13bb574edf98a752055f3d813c, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T02:34:51,651 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/B/faa50f8a627c44e4ac45e90bdc3c7932 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/faa50f8a627c44e4ac45e90bdc3c7932 2024-12-03T02:34:51,655 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/faa50f8a627c44e4ac45e90bdc3c7932, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T02:34:51,656 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/.tmp/C/a73c3950ac5046f0a445db679c6c318b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/a73c3950ac5046f0a445db679c6c318b 2024-12-03T02:34:51,660 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/a73c3950ac5046f0a445db679c6c318b, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T02:34:51,661 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ee5b914e88815d61556ce041d081a81b in 879ms, sequenceid=413, compaction requested=false 2024-12-03T02:34:51,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9ca39eda597c422e83de1103fb9b1d84, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/18c68261268b46a8aba8d457f572b700, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bb33555f96ce47d7a7b2cd69f82c625d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/ada7832e135f4649bebfc2a93d965274, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e93311d550d9471d92d1d78ca0d1699b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/709eae36d4114ba5a2e246adfa8ee157, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bc523a35fb844afb90c528b274dc2d3a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2207f5bb069e472da509c172de1ada04, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/be721cd2f0034229889350c762403daf, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a1137e7f84f34ceda038fb65d3b98f89, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a802042ed56e44738a697b7d4c05263d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/591cd13fac8d4a94ba54cc85f94b8045, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bbb493b710fa4c5a80cc807f8da7e1b6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fa1cc4540d0949c49fc1566bed07ea3c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9f86b5bc507f4d2cbb547be0b96bf8d0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/c6343fa187914905ad52c068af2aead8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2d6e3624fb2b4dc98202f4a547b6274a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fc52f55a1e2549b3abcf42c337c99ecf, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/972ddf058b954abdb6445602a0b18077, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/6c86b837fbb04d9d86ee7a20bdf80097, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/70aa6321ee3449719f1257514aeedb08, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/34c44d6832494149a0e968b53235e004, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f9cf9ebc14d8466a8341b5dba2e997aa, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a4d889833ace496491a44bf2b9ef3c58, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/37decca0c05f47bc9db0c527621b0256, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f080a032c00040b19ccfcda069a5b9c4] to archive 2024-12-03T02:34:51,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:34:51,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9ca39eda597c422e83de1103fb9b1d84 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9ca39eda597c422e83de1103fb9b1d84 2024-12-03T02:34:51,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/18c68261268b46a8aba8d457f572b700 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/18c68261268b46a8aba8d457f572b700 2024-12-03T02:34:51,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bb33555f96ce47d7a7b2cd69f82c625d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bb33555f96ce47d7a7b2cd69f82c625d 2024-12-03T02:34:51,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/ada7832e135f4649bebfc2a93d965274 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/ada7832e135f4649bebfc2a93d965274 2024-12-03T02:34:51,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e93311d550d9471d92d1d78ca0d1699b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e93311d550d9471d92d1d78ca0d1699b 2024-12-03T02:34:51,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/709eae36d4114ba5a2e246adfa8ee157 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/709eae36d4114ba5a2e246adfa8ee157 2024-12-03T02:34:51,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bc523a35fb844afb90c528b274dc2d3a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bc523a35fb844afb90c528b274dc2d3a 2024-12-03T02:34:51,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2207f5bb069e472da509c172de1ada04 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2207f5bb069e472da509c172de1ada04 2024-12-03T02:34:51,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/be721cd2f0034229889350c762403daf to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/be721cd2f0034229889350c762403daf 2024-12-03T02:34:51,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a1137e7f84f34ceda038fb65d3b98f89 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a1137e7f84f34ceda038fb65d3b98f89 2024-12-03T02:34:51,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a802042ed56e44738a697b7d4c05263d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a802042ed56e44738a697b7d4c05263d 2024-12-03T02:34:51,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/591cd13fac8d4a94ba54cc85f94b8045 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/591cd13fac8d4a94ba54cc85f94b8045 2024-12-03T02:34:51,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bbb493b710fa4c5a80cc807f8da7e1b6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/bbb493b710fa4c5a80cc807f8da7e1b6 2024-12-03T02:34:51,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fa1cc4540d0949c49fc1566bed07ea3c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fa1cc4540d0949c49fc1566bed07ea3c 2024-12-03T02:34:51,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9f86b5bc507f4d2cbb547be0b96bf8d0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/9f86b5bc507f4d2cbb547be0b96bf8d0 2024-12-03T02:34:51,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/c6343fa187914905ad52c068af2aead8 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/c6343fa187914905ad52c068af2aead8 2024-12-03T02:34:51,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2d6e3624fb2b4dc98202f4a547b6274a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/2d6e3624fb2b4dc98202f4a547b6274a 2024-12-03T02:34:51,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fc52f55a1e2549b3abcf42c337c99ecf to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/fc52f55a1e2549b3abcf42c337c99ecf 2024-12-03T02:34:51,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/972ddf058b954abdb6445602a0b18077 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/972ddf058b954abdb6445602a0b18077 2024-12-03T02:34:51,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/6c86b837fbb04d9d86ee7a20bdf80097 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/6c86b837fbb04d9d86ee7a20bdf80097 2024-12-03T02:34:51,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/70aa6321ee3449719f1257514aeedb08 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/70aa6321ee3449719f1257514aeedb08 2024-12-03T02:34:51,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/34c44d6832494149a0e968b53235e004 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/34c44d6832494149a0e968b53235e004 2024-12-03T02:34:51,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f9cf9ebc14d8466a8341b5dba2e997aa to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f9cf9ebc14d8466a8341b5dba2e997aa 2024-12-03T02:34:51,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a4d889833ace496491a44bf2b9ef3c58 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/a4d889833ace496491a44bf2b9ef3c58 2024-12-03T02:34:51,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/37decca0c05f47bc9db0c527621b0256 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/37decca0c05f47bc9db0c527621b0256 2024-12-03T02:34:51,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f080a032c00040b19ccfcda069a5b9c4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/f080a032c00040b19ccfcda069a5b9c4 2024-12-03T02:34:51,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/36067541f3544a588c56b7cf4f1361f2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a9f9e73b43a745a6b7bfb5d3362100dc, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21db5e2714d6431c83b5a632acd7c347, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/766ea15750c0464bafb478c9a6e6b45c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/fec5f9b8d213452eb8bab359a8ef359b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/eb886812051c4594a28c225f495cc348, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a4c2d17bd3a0464cbdcbd8023c8ef7a4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9c945e55cec0400497d59d0ef575655d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/7bb53f21befd44749f0e53b2d3d0a177, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/12f40b2f19034ca5b2b1c78cfe2bd687, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/89d843fd679e4299a7b5c23c1f3f5c2f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/64142b41cda74167850bbe779ef2705e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/e33551509f2544ac8ff6e2569db1f75e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/bb3c42a1ffe041ac91bff103f29e7ee5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/29195d21b7874808b2d8be092d7910d8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/b8112e72778b4dde82dee70e6ae45f63, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/df6973bca1f647aa89fde29874fe6402, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a61b99b0fa644ffeb456320d3ff984ca, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ee972ebfcb6e454c9f950c75fec6ebfe, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/abb041c9dfc74e84bd121abe1e3e12a9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ef707bd583d6472bad6a4ca950a44fb9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a5b511d5c2e547369d36d11bcdebfc7d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/d6bf8e4acd1348699bbe8499c35db71f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/702ac4ac6d99455bbc1b99c35255b485, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9442425b79f943ba9c9e9f35ef708b48, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21c258e4fd494cd2be4cc59aefae571e] to archive 2024-12-03T02:34:51,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:34:51,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/36067541f3544a588c56b7cf4f1361f2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/36067541f3544a588c56b7cf4f1361f2 2024-12-03T02:34:51,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a9f9e73b43a745a6b7bfb5d3362100dc to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a9f9e73b43a745a6b7bfb5d3362100dc 2024-12-03T02:34:51,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21db5e2714d6431c83b5a632acd7c347 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21db5e2714d6431c83b5a632acd7c347 2024-12-03T02:34:51,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/766ea15750c0464bafb478c9a6e6b45c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/766ea15750c0464bafb478c9a6e6b45c 2024-12-03T02:34:51,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/fec5f9b8d213452eb8bab359a8ef359b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/fec5f9b8d213452eb8bab359a8ef359b 2024-12-03T02:34:51,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/eb886812051c4594a28c225f495cc348 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/eb886812051c4594a28c225f495cc348 2024-12-03T02:34:51,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a4c2d17bd3a0464cbdcbd8023c8ef7a4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a4c2d17bd3a0464cbdcbd8023c8ef7a4 2024-12-03T02:34:51,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9c945e55cec0400497d59d0ef575655d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9c945e55cec0400497d59d0ef575655d 2024-12-03T02:34:51,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/7bb53f21befd44749f0e53b2d3d0a177 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/7bb53f21befd44749f0e53b2d3d0a177 2024-12-03T02:34:51,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/12f40b2f19034ca5b2b1c78cfe2bd687 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/12f40b2f19034ca5b2b1c78cfe2bd687 2024-12-03T02:34:51,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/89d843fd679e4299a7b5c23c1f3f5c2f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/89d843fd679e4299a7b5c23c1f3f5c2f 2024-12-03T02:34:51,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/64142b41cda74167850bbe779ef2705e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/64142b41cda74167850bbe779ef2705e 2024-12-03T02:34:51,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/e33551509f2544ac8ff6e2569db1f75e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/e33551509f2544ac8ff6e2569db1f75e 2024-12-03T02:34:51,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/bb3c42a1ffe041ac91bff103f29e7ee5 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/bb3c42a1ffe041ac91bff103f29e7ee5 2024-12-03T02:34:51,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/29195d21b7874808b2d8be092d7910d8 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/29195d21b7874808b2d8be092d7910d8 2024-12-03T02:34:51,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/b8112e72778b4dde82dee70e6ae45f63 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/b8112e72778b4dde82dee70e6ae45f63 2024-12-03T02:34:51,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/df6973bca1f647aa89fde29874fe6402 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/df6973bca1f647aa89fde29874fe6402 2024-12-03T02:34:51,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a61b99b0fa644ffeb456320d3ff984ca to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a61b99b0fa644ffeb456320d3ff984ca 2024-12-03T02:34:51,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ee972ebfcb6e454c9f950c75fec6ebfe to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ee972ebfcb6e454c9f950c75fec6ebfe 2024-12-03T02:34:51,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/abb041c9dfc74e84bd121abe1e3e12a9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/abb041c9dfc74e84bd121abe1e3e12a9 2024-12-03T02:34:51,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ef707bd583d6472bad6a4ca950a44fb9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/ef707bd583d6472bad6a4ca950a44fb9 2024-12-03T02:34:51,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a5b511d5c2e547369d36d11bcdebfc7d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/a5b511d5c2e547369d36d11bcdebfc7d 2024-12-03T02:34:51,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/d6bf8e4acd1348699bbe8499c35db71f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/d6bf8e4acd1348699bbe8499c35db71f 2024-12-03T02:34:51,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/702ac4ac6d99455bbc1b99c35255b485 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/702ac4ac6d99455bbc1b99c35255b485 2024-12-03T02:34:51,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9442425b79f943ba9c9e9f35ef708b48 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/9442425b79f943ba9c9e9f35ef708b48 2024-12-03T02:34:51,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21c258e4fd494cd2be4cc59aefae571e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/21c258e4fd494cd2be4cc59aefae571e 2024-12-03T02:34:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T02:34:51,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/069f6326c077473aaa28568b68f727df, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/cdfa7dafbd754eef87b0eb43cb0f3301, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/417df89672c54b209bbfd0901f690fda, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/95ccd58bae344a6daa7068f50f7d97a6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0c23833558304fa18f389e57c0f1b841, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/a3c77796348d41b0b6a6fb69c5be824f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/b1acee84cb9948438b16a3cbfb2b3051, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1f6e33234b0148fbb105bc6f9abd9ed9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7c300bd631ed4953afd0869583a7706f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/aaecb1bb12694509b88f66d5dd94344f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0f65416c93604a7fbe1088ee287a6a88, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/975466afe8f443bf9c8165648c7d11cf, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1eb53a6d73ff436da1a69e7b495d58f4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/77b30071108940c494867b902f025eab, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/e60901f5aca94a629b7d0b711d6ab23f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/5a3f05c3b73f40cda2d7e5daaf7f325e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/886d024c9db447a89d85640e867888b8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2e80ad22f10f483e8c1c933636d4b22c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/380b446827c54273976e1e8976d9f131, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/bef548fd92af49e4bcb83bb258d39f56, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/d5cda8ba25284251aa74934ca436eee2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/eeb6e41c01fc4702bf69011b2fa57b82, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2d5add15cb2b48cbbe828412b3fdc052, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7ea526ece1014b50b1d76ebe22d86572, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/8bc1fde728b04b87bd61af3bda293d12, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2fcecab33d72489e8a5bd65bd2079492] to archive 2024-12-03T02:34:51,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:34:51,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/069f6326c077473aaa28568b68f727df to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/069f6326c077473aaa28568b68f727df 2024-12-03T02:34:51,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/cdfa7dafbd754eef87b0eb43cb0f3301 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/cdfa7dafbd754eef87b0eb43cb0f3301 2024-12-03T02:34:51,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/417df89672c54b209bbfd0901f690fda to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/417df89672c54b209bbfd0901f690fda 2024-12-03T02:34:51,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/95ccd58bae344a6daa7068f50f7d97a6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/95ccd58bae344a6daa7068f50f7d97a6 2024-12-03T02:34:51,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0c23833558304fa18f389e57c0f1b841 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0c23833558304fa18f389e57c0f1b841 2024-12-03T02:34:51,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/a3c77796348d41b0b6a6fb69c5be824f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/a3c77796348d41b0b6a6fb69c5be824f 2024-12-03T02:34:51,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/b1acee84cb9948438b16a3cbfb2b3051 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/b1acee84cb9948438b16a3cbfb2b3051 2024-12-03T02:34:51,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1f6e33234b0148fbb105bc6f9abd9ed9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1f6e33234b0148fbb105bc6f9abd9ed9 2024-12-03T02:34:51,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7c300bd631ed4953afd0869583a7706f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7c300bd631ed4953afd0869583a7706f 2024-12-03T02:34:51,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/aaecb1bb12694509b88f66d5dd94344f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/aaecb1bb12694509b88f66d5dd94344f 2024-12-03T02:34:51,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0f65416c93604a7fbe1088ee287a6a88 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/0f65416c93604a7fbe1088ee287a6a88 2024-12-03T02:34:51,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/975466afe8f443bf9c8165648c7d11cf to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/975466afe8f443bf9c8165648c7d11cf 2024-12-03T02:34:51,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1eb53a6d73ff436da1a69e7b495d58f4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/1eb53a6d73ff436da1a69e7b495d58f4 2024-12-03T02:34:51,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/77b30071108940c494867b902f025eab to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/77b30071108940c494867b902f025eab 2024-12-03T02:34:51,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/e60901f5aca94a629b7d0b711d6ab23f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/e60901f5aca94a629b7d0b711d6ab23f 2024-12-03T02:34:51,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/5a3f05c3b73f40cda2d7e5daaf7f325e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/5a3f05c3b73f40cda2d7e5daaf7f325e 2024-12-03T02:34:51,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/886d024c9db447a89d85640e867888b8 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/886d024c9db447a89d85640e867888b8 2024-12-03T02:34:51,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2e80ad22f10f483e8c1c933636d4b22c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2e80ad22f10f483e8c1c933636d4b22c 2024-12-03T02:34:51,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/380b446827c54273976e1e8976d9f131 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/380b446827c54273976e1e8976d9f131 2024-12-03T02:34:51,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/bef548fd92af49e4bcb83bb258d39f56 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/bef548fd92af49e4bcb83bb258d39f56 2024-12-03T02:34:51,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/d5cda8ba25284251aa74934ca436eee2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/d5cda8ba25284251aa74934ca436eee2 2024-12-03T02:34:51,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/eeb6e41c01fc4702bf69011b2fa57b82 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/eeb6e41c01fc4702bf69011b2fa57b82 2024-12-03T02:34:51,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2d5add15cb2b48cbbe828412b3fdc052 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2d5add15cb2b48cbbe828412b3fdc052 2024-12-03T02:34:51,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7ea526ece1014b50b1d76ebe22d86572 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/7ea526ece1014b50b1d76ebe22d86572 2024-12-03T02:34:51,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/8bc1fde728b04b87bd61af3bda293d12 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/8bc1fde728b04b87bd61af3bda293d12 2024-12-03T02:34:51,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2fcecab33d72489e8a5bd65bd2079492 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/2fcecab33d72489e8a5bd65bd2079492 2024-12-03T02:34:51,843 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/recovered.edits/416.seqid, newMaxSeqId=416, maxSeqId=1 2024-12-03T02:34:51,844 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. 2024-12-03T02:34:51,844 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for ee5b914e88815d61556ce041d081a81b: Waiting for close lock at 1733193290782Running coprocessor pre-close hooks at 1733193290782Disabling compacts and flushes for region at 1733193290782Disabling writes for close at 1733193290782Obtaining lock to block concurrent updates at 1733193290783 (+1 ms)Preparing flush snapshotting stores in ee5b914e88815d61556ce041d081a81b at 1733193290783Finished memstore snapshotting TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733193290783Flushing stores of TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b. at 1733193290784 (+1 ms)Flushing ee5b914e88815d61556ce041d081a81b/A: creating writer at 1733193290784Flushing ee5b914e88815d61556ce041d081a81b/A: appending metadata at 1733193290788 (+4 ms)Flushing ee5b914e88815d61556ce041d081a81b/A: closing flushed file at 1733193290788Flushing ee5b914e88815d61556ce041d081a81b/B: creating writer at 1733193290808 (+20 ms)Flushing ee5b914e88815d61556ce041d081a81b/B: appending metadata at 1733193290812 (+4 ms)Flushing ee5b914e88815d61556ce041d081a81b/B: closing flushed file at 1733193290812Flushing ee5b914e88815d61556ce041d081a81b/C: creating writer at 1733193291233 (+421 ms)Flushing ee5b914e88815d61556ce041d081a81b/C: appending metadata at 1733193291235 (+2 ms)Flushing ee5b914e88815d61556ce041d081a81b/C: closing flushed file at 1733193291235Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67120af3: reopening flushed file at 1733193291645 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bfc6efc: reopening flushed file at 1733193291650 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6081555b: reopening flushed file at 1733193291655 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ee5b914e88815d61556ce041d081a81b in 879ms, sequenceid=413, compaction requested=false at 1733193291661 (+6 ms)Writing region close event to WAL at 1733193291839 (+178 ms)Running coprocessor post-close hooks at 1733193291844 (+5 ms)Closed at 1733193291844 2024-12-03T02:34:51,846 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:51,846 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=ee5b914e88815d61556ce041d081a81b, regionState=CLOSED 2024-12-03T02:34:51,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:34:51,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-03T02:34:51,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseRegionProcedure ee5b914e88815d61556ce041d081a81b, server=64035cbb0b46,38721,1733193202350 in 1.2210 sec 2024-12-03T02:34:51,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-12-03T02:34:51,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ee5b914e88815d61556ce041d081a81b, UNASSIGN in 1.2290 sec 2024-12-03T02:34:51,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-03T02:34:51,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2340 sec 2024-12-03T02:34:51,860 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193291860"}]},"ts":"1733193291860"} 2024-12-03T02:34:51,862 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T02:34:51,862 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T02:34:51,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2450 sec 2024-12-03T02:34:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T02:34:52,765 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:52,766 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T02:34:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:52,767 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T02:34:52,768 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=76, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:52,769 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:52,771 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/recovered.edits] 2024-12-03T02:34:52,774 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/047809ec094941c189b6c5f793de7bd6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/047809ec094941c189b6c5f793de7bd6 2024-12-03T02:34:52,775 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e5d7cb13bb574edf98a752055f3d813c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/A/e5d7cb13bb574edf98a752055f3d813c 2024-12-03T02:34:52,777 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/38e8aac1c55f4858af87f3293007c8b3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/38e8aac1c55f4858af87f3293007c8b3 2024-12-03T02:34:52,778 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/faa50f8a627c44e4ac45e90bdc3c7932 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/B/faa50f8a627c44e4ac45e90bdc3c7932 2024-12-03T02:34:52,780 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/6d1225c3d49d4c49abc00eadbe40e14b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/6d1225c3d49d4c49abc00eadbe40e14b 2024-12-03T02:34:52,781 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/a73c3950ac5046f0a445db679c6c318b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/C/a73c3950ac5046f0a445db679c6c318b 2024-12-03T02:34:52,783 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/recovered.edits/416.seqid to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b/recovered.edits/416.seqid 2024-12-03T02:34:52,784 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/ee5b914e88815d61556ce041d081a81b 2024-12-03T02:34:52,784 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T02:34:52,789 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=76, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:52,792 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T02:34:52,794 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T02:34:52,795 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=76, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:52,795 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T02:34:52,795 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733193292795"}]},"ts":"9223372036854775807"} 2024-12-03T02:34:52,797 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T02:34:52,797 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ee5b914e88815d61556ce041d081a81b, NAME => 'TestAcidGuarantees,,1733193261789.ee5b914e88815d61556ce041d081a81b.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T02:34:52,797 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T02:34:52,797 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733193292797"}]},"ts":"9223372036854775807"} 2024-12-03T02:34:52,799 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T02:34:52,799 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=76, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:52,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-12-03T02:34:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T02:34:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T02:34:52,877 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:52,883 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 251), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 802) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2456 (was 2390) - AvailableMemoryMB LEAK? - 2024-12-03T02:34:52,894 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=2455 2024-12-03T02:34:52,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T02:34:52,895 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:34:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:52,897 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T02:34:52,898 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:52,898 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 77 2024-12-03T02:34:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-03T02:34:52,899 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T02:34:52,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742142_1318 (size=1061) 2024-12-03T02:34:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-03T02:34:53,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-03T02:34:53,305 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37 2024-12-03T02:34:53,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742143_1319 (size=53) 2024-12-03T02:34:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-03T02:34:53,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:34:53,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 309d0efd241d9334329aa843e6c8f6f7, disabling compactions & flushes 2024-12-03T02:34:53,718 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:53,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:53,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. after waiting 0 ms 2024-12-03T02:34:53,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:53,718 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:53,718 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 309d0efd241d9334329aa843e6c8f6f7: Waiting for close lock at 1733193293718Disabling compacts and flushes for region at 1733193293718Disabling writes for close at 1733193293718Writing region close event to WAL at 1733193293718Closed at 1733193293718 2024-12-03T02:34:53,720 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T02:34:53,720 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733193293720"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733193293720"}]},"ts":"1733193293720"} 2024-12-03T02:34:53,722 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T02:34:53,723 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T02:34:53,723 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193293723"}]},"ts":"1733193293723"} 2024-12-03T02:34:53,724 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T02:34:53,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, ASSIGN}] 2024-12-03T02:34:53,731 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, ASSIGN 2024-12-03T02:34:53,732 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, ASSIGN; state=OFFLINE, location=64035cbb0b46,38721,1733193202350; forceNewPlan=false, retain=false 2024-12-03T02:34:53,882 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=309d0efd241d9334329aa843e6c8f6f7, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:53,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, ASSIGN because future has completed 2024-12-03T02:34:53,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:34:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-03T02:34:54,048 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:54,048 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:34:54,049 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,049 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:34:54,049 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,049 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,052 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,053 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:34:54,053 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 309d0efd241d9334329aa843e6c8f6f7 columnFamilyName A 2024-12-03T02:34:54,053 DEBUG [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:54,054 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(327): Store=309d0efd241d9334329aa843e6c8f6f7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:34:54,054 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,056 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:34:54,056 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 309d0efd241d9334329aa843e6c8f6f7 columnFamilyName B 2024-12-03T02:34:54,056 DEBUG [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:54,057 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(327): Store=309d0efd241d9334329aa843e6c8f6f7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:34:54,057 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,058 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:34:54,058 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 309d0efd241d9334329aa843e6c8f6f7 columnFamilyName C 2024-12-03T02:34:54,058 DEBUG [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:54,059 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(327): Store=309d0efd241d9334329aa843e6c8f6f7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:34:54,059 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:54,059 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,064 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,064 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,065 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,065 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,066 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:34:54,068 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,073 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:34:54,076 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 309d0efd241d9334329aa843e6c8f6f7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64391475, jitterRate=-0.04049225151538849}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:34:54,076 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:54,076 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 309d0efd241d9334329aa843e6c8f6f7: Running coprocessor pre-open hook at 1733193294049Writing region info on filesystem at 1733193294049Initializing all the Stores at 1733193294051 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193294051Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193294052 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193294052Cleaning up temporary data from old regions at 1733193294065 (+13 ms)Running coprocessor post-open hooks at 1733193294076 (+11 ms)Region opened successfully at 1733193294076 2024-12-03T02:34:54,077 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., pid=79, masterSystemTime=1733193294038 2024-12-03T02:34:54,079 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:54,079 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:54,082 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=309d0efd241d9334329aa843e6c8f6f7, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:54,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:34:54,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-03T02:34:54,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 in 201 msec 2024-12-03T02:34:54,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-03T02:34:54,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, ASSIGN in 363 msec 2024-12-03T02:34:54,091 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T02:34:54,091 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193294091"}]},"ts":"1733193294091"} 2024-12-03T02:34:54,092 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T02:34:54,094 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T02:34:54,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1990 sec 2024-12-03T02:34:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-03T02:34:55,046 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:55,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:55,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:55,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:55,049 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4893478e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:55,052 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd5760d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:55,054 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:55,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:55,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T02:34:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 64035cbb0b46,44739,1733193201066 2024-12-03T02:34:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64d32ca3 2024-12-03T02:34:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T02:34:55,071 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T02:34:55,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T02:34:55,073 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:34:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T02:34:55,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742144_1320 (size=1097) 2024-12-03T02:34:55,496 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T02:34:55,496 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T02:34:55,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T02:34:55,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, REOPEN/MOVE}] 2024-12-03T02:34:55,503 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, REOPEN/MOVE 2024-12-03T02:34:55,504 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=309d0efd241d9334329aa843e6c8f6f7, regionState=CLOSING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:55,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, REOPEN/MOVE because future has completed 2024-12-03T02:34:55,506 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:34:55,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:34:55,659 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,659 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:34:55,660 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 309d0efd241d9334329aa843e6c8f6f7, disabling compactions & flushes 2024-12-03T02:34:55,660 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:55,660 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:55,660 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. after waiting 0 ms 2024-12-03T02:34:55,660 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:55,663 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T02:34:55,664 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:55,664 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 309d0efd241d9334329aa843e6c8f6f7: Waiting for close lock at 1733193295660Running coprocessor pre-close hooks at 1733193295660Disabling compacts and flushes for region at 1733193295660Disabling writes for close at 1733193295660Writing region close event to WAL at 1733193295660Running coprocessor post-close hooks at 1733193295664 (+4 ms)Closed at 1733193295664 2024-12-03T02:34:55,664 WARN [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionServer(3299): Not adding moved region record: 309d0efd241d9334329aa843e6c8f6f7 to self. 2024-12-03T02:34:55,666 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,667 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=309d0efd241d9334329aa843e6c8f6f7, regionState=CLOSED 2024-12-03T02:34:55,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:34:55,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-03T02:34:55,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 in 164 msec 2024-12-03T02:34:55,675 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, REOPEN/MOVE; state=CLOSED, location=64035cbb0b46,38721,1733193202350; forceNewPlan=false, retain=true 2024-12-03T02:34:55,825 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=309d0efd241d9334329aa843e6c8f6f7, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:55,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, REOPEN/MOVE because future has completed 2024-12-03T02:34:55,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:34:55,985 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:55,985 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7752): Opening region: {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:34:55,985 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,985 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:34:55,985 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7794): checking encryption for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,985 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7797): checking classloading for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,987 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,988 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:34:55,988 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 309d0efd241d9334329aa843e6c8f6f7 columnFamilyName A 2024-12-03T02:34:55,989 DEBUG [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:55,990 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(327): Store=309d0efd241d9334329aa843e6c8f6f7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:34:55,990 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,991 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:34:55,991 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 309d0efd241d9334329aa843e6c8f6f7 columnFamilyName B 2024-12-03T02:34:55,991 DEBUG [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:55,992 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(327): Store=309d0efd241d9334329aa843e6c8f6f7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:34:55,992 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,992 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:34:55,992 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 309d0efd241d9334329aa843e6c8f6f7 columnFamilyName C 2024-12-03T02:34:55,993 DEBUG [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:55,993 INFO [StoreOpener-309d0efd241d9334329aa843e6c8f6f7-1 {}] regionserver.HStore(327): Store=309d0efd241d9334329aa843e6c8f6f7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:34:55,993 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:55,993 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1038): replaying wal for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,994 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,994 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,995 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1048): stopping wal replay for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,995 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1060): Cleaning up temporary data for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,995 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:34:55,996 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1093): writing seq id for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,997 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1114): Opened 309d0efd241d9334329aa843e6c8f6f7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67680576, jitterRate=0.008519172668457031}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:34:55,997 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:55,998 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1006): Region open journal for 309d0efd241d9334329aa843e6c8f6f7: Running coprocessor pre-open hook at 1733193295986Writing region info on filesystem at 1733193295986Initializing all the Stores at 1733193295986Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193295986Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193295987 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193295987Cleaning up temporary data from old regions at 1733193295995 (+8 ms)Running coprocessor post-open hooks at 1733193295997 (+2 ms)Region opened successfully at 1733193295998 (+1 ms) 2024-12-03T02:34:55,998 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., pid=84, masterSystemTime=1733193295982 2024-12-03T02:34:56,000 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,000 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,000 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=309d0efd241d9334329aa843e6c8f6f7, regionState=OPEN, openSeqNum=5, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:34:56,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-12-03T02:34:56,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 in 174 msec 2024-12-03T02:34:56,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-03T02:34:56,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, REOPEN/MOVE in 501 msec 2024-12-03T02:34:56,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-03T02:34:56,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 506 msec 2024-12-03T02:34:56,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 936 msec 2024-12-03T02:34:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T02:34:56,012 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:34:56,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,014 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@548e1aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,016 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c6c35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56cae012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bcd2d6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e49c72c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6b1bc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,024 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0b76bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bbb3c81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,027 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4f9978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,028 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19443910, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,030 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345f79b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,031 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342fbbf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,033 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dca7ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,034 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@543f4eae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,035 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,036 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c711571, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,037 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40754c42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,039 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11f426ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,040 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da5cd64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:34:56,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:34:56,042 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:34:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:34:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:34:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d81fe85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:34:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:34:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:34:56,043 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:34:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2e7188, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:34:56,044 DEBUG [Thread-1480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,044 DEBUG [Thread-1482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,044 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:34:56,044 DEBUG [Thread-1469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,044 DEBUG [Thread-1473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,044 DEBUG [Thread-1478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,045 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-12-03T02:34:56,045 DEBUG [Thread-1467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,046 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:34:56,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,046 DEBUG [Thread-1465 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,047 DEBUG [Thread-1471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,047 DEBUG [Thread-1484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:34:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,049 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:34:56,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:34:56,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:34:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:34:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:34:56,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:34:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:34:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:56,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:34:56,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:34:56,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:56,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:34:56,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:56,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:34:56,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193356105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036ffb26f66cb74423a710452a3f349ccb_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193296066/Put/seqid=0 2024-12-03T02:34:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193356106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193356106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193356107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193356109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742145_1321 (size=14594) 2024-12-03T02:34:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:34:56,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193356215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193356224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193356224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193356225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193356231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:56,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:56,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:34:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193356427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193356435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193356435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193356435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193356444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:56,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:56,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,537 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:56,541 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036ffb26f66cb74423a710452a3f349ccb_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036ffb26f66cb74423a710452a3f349ccb_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:56,542 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9642949b3b024bb7980c4fabf713185a, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:34:56,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9642949b3b024bb7980c4fabf713185a is 175, key is test_row_0/A:col10/1733193296066/Put/seqid=0 2024-12-03T02:34:56,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742146_1322 (size=39549) 2024-12-03T02:34:56,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9642949b3b024bb7980c4fabf713185a 2024-12-03T02:34:56,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/c48a0c08b1504a4da2a8727b9fe174d3 is 50, key is test_row_0/B:col10/1733193296066/Put/seqid=0 2024-12-03T02:34:56,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742147_1323 (size=12001) 2024-12-03T02:34:56,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:34:56,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193356745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193356745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193356748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193356756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193356760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:56,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:56,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:56,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:56,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:57,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:57,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:57,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:57,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/c48a0c08b1504a4da2a8727b9fe174d3 2024-12-03T02:34:57,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/b11b08ae745542aab2a5cc93faeea471 is 50, key is test_row_0/C:col10/1733193296066/Put/seqid=0 2024-12-03T02:34:57,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742148_1324 (size=12001) 2024-12-03T02:34:57,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:57,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:34:57,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193357265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193357265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:57,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193357275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:57,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193357275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:57,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193357276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:57,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:57,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:57,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:57,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:57,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:57,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,400 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:34:57,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:57,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:34:57,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/b11b08ae745542aab2a5cc93faeea471 2024-12-03T02:34:57,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9642949b3b024bb7980c4fabf713185a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9642949b3b024bb7980c4fabf713185a 2024-12-03T02:34:57,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9642949b3b024bb7980c4fabf713185a, entries=200, sequenceid=16, filesize=38.6 K 2024-12-03T02:34:57,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/c48a0c08b1504a4da2a8727b9fe174d3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c48a0c08b1504a4da2a8727b9fe174d3 2024-12-03T02:34:57,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c48a0c08b1504a4da2a8727b9fe174d3, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T02:34:57,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/b11b08ae745542aab2a5cc93faeea471 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b11b08ae745542aab2a5cc93faeea471 2024-12-03T02:34:57,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b11b08ae745542aab2a5cc93faeea471, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T02:34:57,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 309d0efd241d9334329aa843e6c8f6f7 in 1498ms, sequenceid=16, compaction requested=false 2024-12-03T02:34:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:34:57,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-12-03T02:34:57,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:57,635 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T02:34:57,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:34:57,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:57,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:34:57,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:57,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:34:57,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:57,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c845adf92f5b41e4ba54a54eaa7c7cf5_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193296094/Put/seqid=0 2024-12-03T02:34:57,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742149_1325 (size=12154) 2024-12-03T02:34:57,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:57,773 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c845adf92f5b41e4ba54a54eaa7c7cf5_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c845adf92f5b41e4ba54a54eaa7c7cf5_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9ae2def0a1804fecb899c4aa9cd1de23, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:34:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9ae2def0a1804fecb899c4aa9cd1de23 is 175, key is test_row_0/A:col10/1733193296094/Put/seqid=0 2024-12-03T02:34:57,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742150_1326 (size=30955) 2024-12-03T02:34:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:34:58,242 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9ae2def0a1804fecb899c4aa9cd1de23 2024-12-03T02:34:58,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f35b753309594f32b18bff2b8b765bfa is 50, key is test_row_0/B:col10/1733193296094/Put/seqid=0 2024-12-03T02:34:58,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:34:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:58,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742151_1327 (size=12001) 2024-12-03T02:34:58,352 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f35b753309594f32b18bff2b8b765bfa 2024-12-03T02:34:58,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193358338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193358340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193358340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193358358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193358360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/ca4b0969676e4b44b6accf2636ceb0cb is 50, key is test_row_0/C:col10/1733193296094/Put/seqid=0 2024-12-03T02:34:58,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742152_1328 (size=12001) 2024-12-03T02:34:58,459 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/ca4b0969676e4b44b6accf2636ceb0cb 2024-12-03T02:34:58,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193358467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193358468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9ae2def0a1804fecb899c4aa9cd1de23 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9ae2def0a1804fecb899c4aa9cd1de23 2024-12-03T02:34:58,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193358475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193358494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193358494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,519 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9ae2def0a1804fecb899c4aa9cd1de23, entries=150, sequenceid=36, filesize=30.2 K 2024-12-03T02:34:58,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f35b753309594f32b18bff2b8b765bfa as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f35b753309594f32b18bff2b8b765bfa 2024-12-03T02:34:58,537 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f35b753309594f32b18bff2b8b765bfa, entries=150, sequenceid=36, filesize=11.7 K 2024-12-03T02:34:58,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/ca4b0969676e4b44b6accf2636ceb0cb as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/ca4b0969676e4b44b6accf2636ceb0cb 2024-12-03T02:34:58,552 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/ca4b0969676e4b44b6accf2636ceb0cb, entries=150, sequenceid=36, filesize=11.7 K 2024-12-03T02:34:58,554 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 309d0efd241d9334329aa843e6c8f6f7 in 919ms, sequenceid=36, compaction requested=false 2024-12-03T02:34:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:34:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:34:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-03T02:34:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-12-03T02:34:58,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-03T02:34:58,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5060 sec 2024-12-03T02:34:58,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 2.5180 sec 2024-12-03T02:34:58,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T02:34:58,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:34:58,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:58,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:34:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:34:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:34:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:58,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203102ddc2f8f98463daecd6df813ffbae7_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193298338/Put/seqid=0 2024-12-03T02:34:58,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193358748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193358748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742153_1329 (size=14594) 2024-12-03T02:34:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193358761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193358764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193358767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193358875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193358877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193358885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193358886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:58,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193358886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193359087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193359095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193359095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193359105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193359105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,167 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:34:59,172 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203102ddc2f8f98463daecd6df813ffbae7_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203102ddc2f8f98463daecd6df813ffbae7_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:34:59,174 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/d9b5faf5fb184ddbb4f9393860deb757, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:34:59,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/d9b5faf5fb184ddbb4f9393860deb757 is 175, key is test_row_0/A:col10/1733193298338/Put/seqid=0 2024-12-03T02:34:59,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742154_1330 (size=39549) 2024-12-03T02:34:59,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193359405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193359416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193359417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193359418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193359425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,585 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/d9b5faf5fb184ddbb4f9393860deb757 2024-12-03T02:34:59,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/81aed039ccee4bd1b4196a4bdbf32075 is 50, key is test_row_0/B:col10/1733193298338/Put/seqid=0 2024-12-03T02:34:59,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742155_1331 (size=12001) 2024-12-03T02:34:59,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/81aed039ccee4bd1b4196a4bdbf32075 2024-12-03T02:34:59,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/0f09894ba4a6415faace401973b9bcdc is 50, key is test_row_0/C:col10/1733193298338/Put/seqid=0 2024-12-03T02:34:59,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742156_1332 (size=12001) 2024-12-03T02:34:59,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193359927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193359955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193359964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193359964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:34:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193359964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:34:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:34:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:34:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:00,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/0f09894ba4a6415faace401973b9bcdc 2024-12-03T02:35:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/d9b5faf5fb184ddbb4f9393860deb757 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/d9b5faf5fb184ddbb4f9393860deb757 2024-12-03T02:35:00,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/d9b5faf5fb184ddbb4f9393860deb757, entries=200, sequenceid=49, filesize=38.6 K 2024-12-03T02:35:00,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/81aed039ccee4bd1b4196a4bdbf32075 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/81aed039ccee4bd1b4196a4bdbf32075 2024-12-03T02:35:00,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/81aed039ccee4bd1b4196a4bdbf32075, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T02:35:00,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/0f09894ba4a6415faace401973b9bcdc as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0f09894ba4a6415faace401973b9bcdc 2024-12-03T02:35:00,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0f09894ba4a6415faace401973b9bcdc, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T02:35:00,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 309d0efd241d9334329aa843e6c8f6f7 in 1486ms, sequenceid=49, compaction requested=true 2024-12-03T02:35:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:00,176 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:00,176 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:00,177 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:00,177 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:00,177 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/A is initiating minor compaction (all files) 2024-12-03T02:35:00,177 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/B is initiating minor compaction (all files) 2024-12-03T02:35:00,177 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/A in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:00,177 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/B in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:00,177 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9642949b3b024bb7980c4fabf713185a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9ae2def0a1804fecb899c4aa9cd1de23, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/d9b5faf5fb184ddbb4f9393860deb757] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=107.5 K 2024-12-03T02:35:00,177 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c48a0c08b1504a4da2a8727b9fe174d3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f35b753309594f32b18bff2b8b765bfa, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/81aed039ccee4bd1b4196a4bdbf32075] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=35.2 K 2024-12-03T02:35:00,177 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:00,177 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9642949b3b024bb7980c4fabf713185a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9ae2def0a1804fecb899c4aa9cd1de23, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/d9b5faf5fb184ddbb4f9393860deb757] 2024-12-03T02:35:00,177 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting c48a0c08b1504a4da2a8727b9fe174d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733193296057 2024-12-03T02:35:00,178 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9642949b3b024bb7980c4fabf713185a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733193296057 2024-12-03T02:35:00,178 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting f35b753309594f32b18bff2b8b765bfa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733193296094 2024-12-03T02:35:00,178 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ae2def0a1804fecb899c4aa9cd1de23, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733193296094 2024-12-03T02:35:00,178 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 81aed039ccee4bd1b4196a4bdbf32075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733193298338 2024-12-03T02:35:00,178 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9b5faf5fb184ddbb4f9393860deb757, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733193298319 2024-12-03T02:35:00,193 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:00,193 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/4f5d125f43024a4286ddcbed9d20ede7 is 50, key is test_row_0/B:col10/1733193298338/Put/seqid=0 2024-12-03T02:35:00,196 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:00,212 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412038d7119eb8aae462e91be0b15390c4a55_309d0efd241d9334329aa843e6c8f6f7 store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:00,213 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412038d7119eb8aae462e91be0b15390c4a55_309d0efd241d9334329aa843e6c8f6f7, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:00,213 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038d7119eb8aae462e91be0b15390c4a55_309d0efd241d9334329aa843e6c8f6f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:00,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742157_1333 (size=12104) 2024-12-03T02:35:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:35:00,235 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:00,236 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-12-03T02:35:00,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742158_1334 (size=4469) 2024-12-03T02:35:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T02:35:00,238 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:00,239 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:00,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T02:35:00,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-12-03T02:35:00,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:00,391 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T02:35:00,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:00,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:00,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:00,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:00,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:00,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:00,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203883f32702c5f4b66b0677c1da706c6c4_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193298759/Put/seqid=0 2024-12-03T02:35:00,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742159_1335 (size=12154) 2024-12-03T02:35:00,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,436 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203883f32702c5f4b66b0677c1da706c6c4_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203883f32702c5f4b66b0677c1da706c6c4_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/2744b5537b1b42c9b2a857b57124abf5, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:00,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/2744b5537b1b42c9b2a857b57124abf5 is 175, key is test_row_0/A:col10/1733193298759/Put/seqid=0 2024-12-03T02:35:00,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742160_1336 (size=30955) 2024-12-03T02:35:00,503 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/2744b5537b1b42c9b2a857b57124abf5 2024-12-03T02:35:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5a22e14b02554dd98f517c6512fc2c46 is 50, key is test_row_0/B:col10/1733193298759/Put/seqid=0 2024-12-03T02:35:00,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742161_1337 (size=12001) 2024-12-03T02:35:00,530 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5a22e14b02554dd98f517c6512fc2c46 2024-12-03T02:35:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T02:35:00,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/7e6cadc0f43c43e79ac2faa17f2c156f is 50, key is test_row_0/C:col10/1733193298759/Put/seqid=0 2024-12-03T02:35:00,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742162_1338 (size=12001) 2024-12-03T02:35:00,592 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/7e6cadc0f43c43e79ac2faa17f2c156f 2024-12-03T02:35:00,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/2744b5537b1b42c9b2a857b57124abf5 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/2744b5537b1b42c9b2a857b57124abf5 2024-12-03T02:35:00,604 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/2744b5537b1b42c9b2a857b57124abf5, entries=150, sequenceid=68, filesize=30.2 K 2024-12-03T02:35:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5a22e14b02554dd98f517c6512fc2c46 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5a22e14b02554dd98f517c6512fc2c46 2024-12-03T02:35:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,613 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5a22e14b02554dd98f517c6512fc2c46, entries=150, sequenceid=68, filesize=11.7 K 2024-12-03T02:35:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/7e6cadc0f43c43e79ac2faa17f2c156f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7e6cadc0f43c43e79ac2faa17f2c156f 2024-12-03T02:35:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,617 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7e6cadc0f43c43e79ac2faa17f2c156f, entries=150, sequenceid=68, filesize=11.7 K 2024-12-03T02:35:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,617 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 309d0efd241d9334329aa843e6c8f6f7 in 226ms, sequenceid=68, compaction requested=true 2024-12-03T02:35:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-03T02:35:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-12-03T02:35:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-03T02:35:00,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 379 msec 2024-12-03T02:35:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 385 msec 2024-12-03T02:35:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,625 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/4f5d125f43024a4286ddcbed9d20ede7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4f5d125f43024a4286ddcbed9d20ede7 2024-12-03T02:35:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,638 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/B of 309d0efd241d9334329aa843e6c8f6f7 into 4f5d125f43024a4286ddcbed9d20ede7(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:00,638 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:00,638 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/B, priority=13, startTime=1733193300176; duration=0sec 2024-12-03T02:35:00,638 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:00,638 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:B 2024-12-03T02:35:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,638 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,640 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,640 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/C is initiating minor compaction (all files) 2024-12-03T02:35:00,641 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/C in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:00,641 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b11b08ae745542aab2a5cc93faeea471, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/ca4b0969676e4b44b6accf2636ceb0cb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0f09894ba4a6415faace401973b9bcdc, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7e6cadc0f43c43e79ac2faa17f2c156f] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=46.9 K 2024-12-03T02:35:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,641 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b11b08ae745542aab2a5cc93faeea471, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733193296057 2024-12-03T02:35:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,642 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4b0969676e4b44b6accf2636ceb0cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733193296094 2024-12-03T02:35:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,642 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f09894ba4a6415faace401973b9bcdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733193298338 2024-12-03T02:35:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,642 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e6cadc0f43c43e79ac2faa17f2c156f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733193298744 2024-12-03T02:35:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,644 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#A#compaction#280 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,645 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/059aa1f936e9424998aac75fe2f851a7 is 175, key is test_row_0/A:col10/1733193298338/Put/seqid=0 2024-12-03T02:35:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,658 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:00,658 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/c0e07afb22594677bd8d79e1129ecc3a is 50, key is test_row_0/C:col10/1733193298759/Put/seqid=0 2024-12-03T02:35:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742163_1339 (size=31058) 2024-12-03T02:35:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742164_1340 (size=12139) 2024-12-03T02:35:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T02:35:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,875 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,880 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-03T02:35:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,889 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T02:35:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,890 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T02:35:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T02:35:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-03T02:35:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-03T02:35:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-03T02:35:01,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 160 msec 2024-12-03T02:35:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 174 msec 2024-12-03T02:35:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,108 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/059aa1f936e9424998aac75fe2f851a7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/059aa1f936e9424998aac75fe2f851a7 2024-12-03T02:35:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,111 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/c0e07afb22594677bd8d79e1129ecc3a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/c0e07afb22594677bd8d79e1129ecc3a 2024-12-03T02:35:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,134 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/A of 309d0efd241d9334329aa843e6c8f6f7 into 059aa1f936e9424998aac75fe2f851a7(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:01,134 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:01,134 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/A, priority=13, startTime=1733193300176; duration=0sec 2024-12-03T02:35:01,134 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:01,134 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:A 2024-12-03T02:35:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,137 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/C of 309d0efd241d9334329aa843e6c8f6f7 into c0e07afb22594677bd8d79e1129ecc3a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:01,137 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:01,137 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/C, priority=12, startTime=1733193300176; duration=0sec 2024-12-03T02:35:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,137 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:01,137 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:C 2024-12-03T02:35:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:35:01,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:01,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:01,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:01,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:01,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:01,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034334d0f33b2f43d583562ac9071e2c0a_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193301084/Put/seqid=0 2024-12-03T02:35:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T02:35:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,206 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-03T02:35:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,210 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:01,211 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:01,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742165_1341 (size=21918) 2024-12-03T02:35:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T02:35:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733193301154, max timestamp=1733193301262Num uniques 859; 2024-12-03T02:35:01,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T02:35:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733193301154, max timestamp=1733193301262Num uniques 859; 2024-12-03T02:35:01,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T02:35:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733193301154, max timestamp=1733193301262Num uniques 859; 2024-12-03T02:35:01,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T02:35:01,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193361277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193361279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193361281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193361284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193361284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T02:35:01,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T02:35:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:01,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:01,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:01,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193361395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193361406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193361406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193361406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193361406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T02:35:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:01,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T02:35:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:01,609 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:01,615 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034334d0f33b2f43d583562ac9071e2c0a_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034334d0f33b2f43d583562ac9071e2c0a_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:01,616 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/f592dc7c718b49228606ae7ed6986f88, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:01,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/f592dc7c718b49228606ae7ed6986f88 is 175, key is test_row_0/A:col10/1733193301084/Put/seqid=0 2024-12-03T02:35:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193361615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193361625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193361626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193361626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193361626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742166_1342 (size=65323) 2024-12-03T02:35:01,677 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/f592dc7c718b49228606ae7ed6986f88 2024-12-03T02:35:01,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T02:35:01,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:01,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:01,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:01,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f73cb920d0a24f1f8d351932a54f595e is 50, key is test_row_0/B:col10/1733193301084/Put/seqid=0 2024-12-03T02:35:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:01,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742167_1343 (size=12001) 2024-12-03T02:35:01,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f73cb920d0a24f1f8d351932a54f595e 2024-12-03T02:35:01,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/749cecfdfca74e8f8fbb0933bb91dcb6 is 50, key is test_row_0/C:col10/1733193301084/Put/seqid=0 2024-12-03T02:35:01,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742168_1344 (size=12001) 2024-12-03T02:35:01,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/749cecfdfca74e8f8fbb0933bb91dcb6 2024-12-03T02:35:01,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/f592dc7c718b49228606ae7ed6986f88 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/f592dc7c718b49228606ae7ed6986f88 2024-12-03T02:35:01,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/f592dc7c718b49228606ae7ed6986f88, entries=350, sequenceid=82, filesize=63.8 K 2024-12-03T02:35:01,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f73cb920d0a24f1f8d351932a54f595e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f73cb920d0a24f1f8d351932a54f595e 2024-12-03T02:35:01,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f73cb920d0a24f1f8d351932a54f595e, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T02:35:01,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/749cecfdfca74e8f8fbb0933bb91dcb6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/749cecfdfca74e8f8fbb0933bb91dcb6 2024-12-03T02:35:01,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/749cecfdfca74e8f8fbb0933bb91dcb6, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T02:35:01,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=161.02 KB/164880 for 309d0efd241d9334329aa843e6c8f6f7 in 683ms, sequenceid=82, compaction requested=true 2024-12-03T02:35:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:01,838 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:01,838 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:01,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:01,839 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127336 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:01,839 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/A is initiating minor compaction (all files) 2024-12-03T02:35:01,839 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/A in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,839 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/059aa1f936e9424998aac75fe2f851a7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/2744b5537b1b42c9b2a857b57124abf5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/f592dc7c718b49228606ae7ed6986f88] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=124.4 K 2024-12-03T02:35:01,839 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,839 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/059aa1f936e9424998aac75fe2f851a7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/2744b5537b1b42c9b2a857b57124abf5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/f592dc7c718b49228606ae7ed6986f88] 2024-12-03T02:35:01,840 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 059aa1f936e9424998aac75fe2f851a7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733193298338 2024-12-03T02:35:01,840 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:01,840 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/B is initiating minor compaction (all files) 2024-12-03T02:35:01,840 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/B in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,840 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4f5d125f43024a4286ddcbed9d20ede7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5a22e14b02554dd98f517c6512fc2c46, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f73cb920d0a24f1f8d351932a54f595e] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=35.3 K 2024-12-03T02:35:01,840 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2744b5537b1b42c9b2a857b57124abf5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733193298744 2024-12-03T02:35:01,841 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f5d125f43024a4286ddcbed9d20ede7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733193298338 2024-12-03T02:35:01,841 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting f592dc7c718b49228606ae7ed6986f88, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733193301065 2024-12-03T02:35:01,841 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a22e14b02554dd98f517c6512fc2c46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733193298744 2024-12-03T02:35:01,841 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting f73cb920d0a24f1f8d351932a54f595e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733193301084 2024-12-03T02:35:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T02:35:01,849 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:01,850 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:01,850 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f7e0136d227e44a9b9e46a556c5c1852 is 50, key is test_row_0/B:col10/1733193301084/Put/seqid=0 2024-12-03T02:35:01,860 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203e991f269509d4165b9e5854f5e2b020f_309d0efd241d9334329aa843e6c8f6f7 store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:01,861 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203e991f269509d4165b9e5854f5e2b020f_309d0efd241d9334329aa843e6c8f6f7, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:01,862 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e991f269509d4165b9e5854f5e2b020f_309d0efd241d9334329aa843e6c8f6f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:01,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T02:35:01,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:01,862 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=161.02 KB heapSize=399.05 KB 2024-12-03T02:35:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:01,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742169_1345 (size=12207) 2024-12-03T02:35:01,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120352fdb4ba1a0749fb9cdaf9ccb6b993b5_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193301280/Put/seqid=0 2024-12-03T02:35:01,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742170_1346 (size=4469) 2024-12-03T02:35:01,889 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#A#compaction#289 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:01,890 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/20a61c4e19e14c6fac1da7861ec6d966 is 175, key is test_row_0/A:col10/1733193301084/Put/seqid=0 2024-12-03T02:35:01,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742171_1347 (size=12154) 2024-12-03T02:35:01,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742172_1348 (size=31161) 2024-12-03T02:35:01,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:01,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193361952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193361955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193361955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193361956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193361959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193362076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193362077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193362077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193362077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193362077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:02,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193362305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193362307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193362307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193362307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193362307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,329 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120352fdb4ba1a0749fb9cdaf9ccb6b993b5_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120352fdb4ba1a0749fb9cdaf9ccb6b993b5_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:02,331 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/20a61c4e19e14c6fac1da7861ec6d966 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/20a61c4e19e14c6fac1da7861ec6d966 2024-12-03T02:35:02,331 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f7e0136d227e44a9b9e46a556c5c1852 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f7e0136d227e44a9b9e46a556c5c1852 2024-12-03T02:35:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/583d2d90a45c451396dc4d8a6a90b259, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:02,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/583d2d90a45c451396dc4d8a6a90b259 is 175, key is test_row_0/A:col10/1733193301280/Put/seqid=0 2024-12-03T02:35:02,355 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/A of 309d0efd241d9334329aa843e6c8f6f7 into 20a61c4e19e14c6fac1da7861ec6d966(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:02,355 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:02,355 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/A, priority=13, startTime=1733193301838; duration=0sec 2024-12-03T02:35:02,355 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:02,355 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:A 2024-12-03T02:35:02,355 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T02:35:02,357 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/B of 309d0efd241d9334329aa843e6c8f6f7 into f7e0136d227e44a9b9e46a556c5c1852(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:02,357 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:02,357 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/B, priority=13, startTime=1733193301838; duration=0sec 2024-12-03T02:35:02,357 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:02,357 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:B 2024-12-03T02:35:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T02:35:02,363 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T02:35:02,364 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T02:35:02,364 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. because compaction request was cancelled 2024-12-03T02:35:02,364 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:C 2024-12-03T02:35:02,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742173_1349 (size=30955) 2024-12-03T02:35:02,383 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/583d2d90a45c451396dc4d8a6a90b259 2024-12-03T02:35:02,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/ff9d874b8cf44bcda04f0e6b6b62bb2b is 50, key is test_row_0/B:col10/1733193301280/Put/seqid=0 2024-12-03T02:35:02,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742174_1350 (size=12001) 2024-12-03T02:35:02,457 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/ff9d874b8cf44bcda04f0e6b6b62bb2b 2024-12-03T02:35:02,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/8f51062029c24545852efe91fad3dc83 is 50, key is test_row_0/C:col10/1733193301280/Put/seqid=0 2024-12-03T02:35:02,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742175_1351 (size=12001) 2024-12-03T02:35:02,494 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/8f51062029c24545852efe91fad3dc83 2024-12-03T02:35:02,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/583d2d90a45c451396dc4d8a6a90b259 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/583d2d90a45c451396dc4d8a6a90b259 2024-12-03T02:35:02,535 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/583d2d90a45c451396dc4d8a6a90b259, entries=150, sequenceid=109, filesize=30.2 K 2024-12-03T02:35:02,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/ff9d874b8cf44bcda04f0e6b6b62bb2b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/ff9d874b8cf44bcda04f0e6b6b62bb2b 2024-12-03T02:35:02,555 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/ff9d874b8cf44bcda04f0e6b6b62bb2b, entries=150, sequenceid=109, filesize=11.7 K 2024-12-03T02:35:02,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/8f51062029c24545852efe91fad3dc83 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8f51062029c24545852efe91fad3dc83 2024-12-03T02:35:02,563 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8f51062029c24545852efe91fad3dc83, entries=150, sequenceid=109, filesize=11.7 K 2024-12-03T02:35:02,566 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~399 KB/408576, currentSize=40.25 KB/41220 for 309d0efd241d9334329aa843e6c8f6f7 in 704ms, sequenceid=109, compaction requested=true 2024-12-03T02:35:02,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:02,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:02,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-03T02:35:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-03T02:35:02,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-03T02:35:02,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3620 sec 2024-12-03T02:35:02,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.3690 sec 2024-12-03T02:35:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:02,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T02:35:02,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:02,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:02,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:02,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:02,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:02,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:02,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cab3e3986114182848704e546403af9_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193302631/Put/seqid=0 2024-12-03T02:35:02,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742176_1352 (size=14594) 2024-12-03T02:35:02,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193362683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193362684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193362685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193362690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193362692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193362814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193362815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193362815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193362815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193362815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193363024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193363024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193363024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193363035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193363035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,083 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:03,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cab3e3986114182848704e546403af9_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cab3e3986114182848704e546403af9_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:03,086 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/842095f305994b4d8887999864a5fd7c, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:03,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/842095f305994b4d8887999864a5fd7c is 175, key is test_row_0/A:col10/1733193302631/Put/seqid=0 2024-12-03T02:35:03,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742177_1353 (size=39549) 2024-12-03T02:35:03,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193363355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193363355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193363355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193363355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193363356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T02:35:03,375 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:03,376 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-03T02:35:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T02:35:03,382 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:03,382 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:03,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T02:35:03,532 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/842095f305994b4d8887999864a5fd7c 2024-12-03T02:35:03,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T02:35:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:03,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:03,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/a51a317df2154abf8adf33a84aa625d0 is 50, key is test_row_0/B:col10/1733193302631/Put/seqid=0 2024-12-03T02:35:03,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742178_1354 (size=12001) 2024-12-03T02:35:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T02:35:03,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T02:35:03,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:03,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:03,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:03,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:03,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:03,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T02:35:03,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:03,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:03,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:03,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193363867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193363868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193363875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193363875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193363875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T02:35:04,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/a51a317df2154abf8adf33a84aa625d0 2024-12-03T02:35:04,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T02:35:04,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:04,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:04,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:04,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:04,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:04,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/7fafe3aebfdd4dbcbb7f43afecaf54fc is 50, key is test_row_0/C:col10/1733193302631/Put/seqid=0 2024-12-03T02:35:04,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742179_1355 (size=12001) 2024-12-03T02:35:04,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/7fafe3aebfdd4dbcbb7f43afecaf54fc 2024-12-03T02:35:04,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/842095f305994b4d8887999864a5fd7c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/842095f305994b4d8887999864a5fd7c 2024-12-03T02:35:04,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/842095f305994b4d8887999864a5fd7c, entries=200, sequenceid=123, filesize=38.6 K 2024-12-03T02:35:04,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/a51a317df2154abf8adf33a84aa625d0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/a51a317df2154abf8adf33a84aa625d0 2024-12-03T02:35:04,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T02:35:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:04,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:04,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:04,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/a51a317df2154abf8adf33a84aa625d0, entries=150, sequenceid=123, filesize=11.7 K 2024-12-03T02:35:04,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/7fafe3aebfdd4dbcbb7f43afecaf54fc as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7fafe3aebfdd4dbcbb7f43afecaf54fc 2024-12-03T02:35:04,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7fafe3aebfdd4dbcbb7f43afecaf54fc, entries=150, sequenceid=123, filesize=11.7 K 2024-12-03T02:35:04,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 309d0efd241d9334329aa843e6c8f6f7 in 1626ms, sequenceid=123, compaction requested=true 2024-12-03T02:35:04,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:35:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T02:35:04,260 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:04,263 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:04,275 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:04,275 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/C is initiating minor compaction (all files) 2024-12-03T02:35:04,275 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/C in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:04,276 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/c0e07afb22594677bd8d79e1129ecc3a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/749cecfdfca74e8f8fbb0933bb91dcb6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8f51062029c24545852efe91fad3dc83, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7fafe3aebfdd4dbcbb7f43afecaf54fc] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=47.0 K 2024-12-03T02:35:04,279 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e07afb22594677bd8d79e1129ecc3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733193298744 2024-12-03T02:35:04,280 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:04,280 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/A is initiating minor compaction (all files) 2024-12-03T02:35:04,280 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/A in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:04,280 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/20a61c4e19e14c6fac1da7861ec6d966, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/583d2d90a45c451396dc4d8a6a90b259, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/842095f305994b4d8887999864a5fd7c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=99.3 K 2024-12-03T02:35:04,280 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:04,280 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/20a61c4e19e14c6fac1da7861ec6d966, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/583d2d90a45c451396dc4d8a6a90b259, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/842095f305994b4d8887999864a5fd7c] 2024-12-03T02:35:04,281 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a61c4e19e14c6fac1da7861ec6d966, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733193301084 2024-12-03T02:35:04,281 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 749cecfdfca74e8f8fbb0933bb91dcb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733193301084 2024-12-03T02:35:04,281 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 583d2d90a45c451396dc4d8a6a90b259, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733193301275 2024-12-03T02:35:04,281 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f51062029c24545852efe91fad3dc83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733193301275 2024-12-03T02:35:04,281 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 842095f305994b4d8887999864a5fd7c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733193301954 2024-12-03T02:35:04,283 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fafe3aebfdd4dbcbb7f43afecaf54fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733193302631 2024-12-03T02:35:04,312 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:04,318 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:04,318 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/3fbd5ead21684506844d75d141c15029 is 50, key is test_row_0/C:col10/1733193302631/Put/seqid=0 2024-12-03T02:35:04,320 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203b3c9abf627a349bc926e0c57a1c83d2e_309d0efd241d9334329aa843e6c8f6f7 store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:04,321 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203b3c9abf627a349bc926e0c57a1c83d2e_309d0efd241d9334329aa843e6c8f6f7, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:04,321 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b3c9abf627a349bc926e0c57a1c83d2e_309d0efd241d9334329aa843e6c8f6f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:04,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T02:35:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:04,357 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T02:35:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:04,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742180_1356 (size=12275) 2024-12-03T02:35:04,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742181_1357 (size=4469) 2024-12-03T02:35:04,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b512cc5aa59e46d194865ace6e801935_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193302685/Put/seqid=0 2024-12-03T02:35:04,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742182_1358 (size=12304) 2024-12-03T02:35:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T02:35:04,781 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/3fbd5ead21684506844d75d141c15029 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/3fbd5ead21684506844d75d141c15029 2024-12-03T02:35:04,785 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/C of 309d0efd241d9334329aa843e6c8f6f7 into 3fbd5ead21684506844d75d141c15029(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:04,785 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:04,785 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/C, priority=12, startTime=1733193304260; duration=0sec 2024-12-03T02:35:04,785 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:04,785 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:C 2024-12-03T02:35:04,785 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:04,785 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:04,786 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/B is initiating minor compaction (all files) 2024-12-03T02:35:04,786 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/B in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:04,786 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f7e0136d227e44a9b9e46a556c5c1852, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/ff9d874b8cf44bcda04f0e6b6b62bb2b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/a51a317df2154abf8adf33a84aa625d0] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=35.4 K 2024-12-03T02:35:04,786 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7e0136d227e44a9b9e46a556c5c1852, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733193301084 2024-12-03T02:35:04,786 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff9d874b8cf44bcda04f0e6b6b62bb2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733193301275 2024-12-03T02:35:04,786 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a51a317df2154abf8adf33a84aa625d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733193302631 2024-12-03T02:35:04,794 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#B#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:04,795 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5d5677613cc14ac29fd066f168bd43db is 50, key is test_row_0/B:col10/1733193302631/Put/seqid=0 2024-12-03T02:35:04,802 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#A#compaction#296 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:04,802 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a91945940b4f42cabb0b5468afbfa69d is 175, key is test_row_0/A:col10/1733193302631/Put/seqid=0 2024-12-03T02:35:04,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742183_1359 (size=12309) 2024-12-03T02:35:04,821 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5d5677613cc14ac29fd066f168bd43db as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5d5677613cc14ac29fd066f168bd43db 2024-12-03T02:35:04,829 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/B of 309d0efd241d9334329aa843e6c8f6f7 into 5d5677613cc14ac29fd066f168bd43db(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:04,829 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:04,829 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/B, priority=13, startTime=1733193304260; duration=0sec 2024-12-03T02:35:04,829 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:04,829 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:B 2024-12-03T02:35:04,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742184_1360 (size=31263) 2024-12-03T02:35:04,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:04,898 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b512cc5aa59e46d194865ace6e801935_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b512cc5aa59e46d194865ace6e801935_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:04,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:04,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/75e429ce07224e508c78d3da93dec440, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:04,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/75e429ce07224e508c78d3da93dec440 is 175, key is test_row_0/A:col10/1733193302685/Put/seqid=0 2024-12-03T02:35:04,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742185_1361 (size=31105) 2024-12-03T02:35:04,908 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/75e429ce07224e508c78d3da93dec440 2024-12-03T02:35:04,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/bc4ccd357e674a5d89dcd52a735593a1 is 50, key is test_row_0/B:col10/1733193302685/Put/seqid=0 2024-12-03T02:35:04,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193364922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:04,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193364923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:04,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193364927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:04,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742186_1362 (size=12151) 2024-12-03T02:35:04,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193364947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:04,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193364965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193365055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193365059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193365059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193365065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193365085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193365266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,280 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a91945940b4f42cabb0b5468afbfa69d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a91945940b4f42cabb0b5468afbfa69d 2024-12-03T02:35:05,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193365275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193365275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193365280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,303 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/A of 309d0efd241d9334329aa843e6c8f6f7 into a91945940b4f42cabb0b5468afbfa69d(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-12-03T02:35:05,303 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:05,303 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/A, priority=13, startTime=1733193304260; duration=1sec 2024-12-03T02:35:05,303 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:05,303 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:A 2024-12-03T02:35:05,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193365295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,352 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/bc4ccd357e674a5d89dcd52a735593a1 2024-12-03T02:35:05,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/519e2f6a4c7d474b8bdc3a1a5079ea8a is 50, key is test_row_0/C:col10/1733193302685/Put/seqid=0 2024-12-03T02:35:05,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742187_1363 (size=12151) 2024-12-03T02:35:05,406 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/519e2f6a4c7d474b8bdc3a1a5079ea8a 2024-12-03T02:35:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/75e429ce07224e508c78d3da93dec440 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/75e429ce07224e508c78d3da93dec440 2024-12-03T02:35:05,421 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/75e429ce07224e508c78d3da93dec440, entries=150, sequenceid=143, filesize=30.4 K 2024-12-03T02:35:05,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/bc4ccd357e674a5d89dcd52a735593a1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/bc4ccd357e674a5d89dcd52a735593a1 2024-12-03T02:35:05,452 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/bc4ccd357e674a5d89dcd52a735593a1, entries=150, sequenceid=143, filesize=11.9 K 2024-12-03T02:35:05,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/519e2f6a4c7d474b8bdc3a1a5079ea8a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/519e2f6a4c7d474b8bdc3a1a5079ea8a 2024-12-03T02:35:05,469 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/519e2f6a4c7d474b8bdc3a1a5079ea8a, entries=150, sequenceid=143, filesize=11.9 K 2024-12-03T02:35:05,470 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 309d0efd241d9334329aa843e6c8f6f7 in 1113ms, sequenceid=143, compaction requested=false 2024-12-03T02:35:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-03T02:35:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-03T02:35:05,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-03T02:35:05,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0900 sec 2024-12-03T02:35:05,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.1080 sec 2024-12-03T02:35:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T02:35:05,535 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:05,535 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-03T02:35:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T02:35:05,537 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:05,538 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:05,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:05,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T02:35:05,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:05,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:05,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:05,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:05,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:05,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:05,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031f3575b58cb046eb932c4e56f2f4a7b3_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193305590/Put/seqid=0 2024-12-03T02:35:05,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742188_1364 (size=14794) 2024-12-03T02:35:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T02:35:05,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193365649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193365651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193365668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193365668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193365669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T02:35:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:05,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:05,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193365775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193365776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193365785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193365785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193365794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T02:35:05,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:05,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:05,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T02:35:05,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:05,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:05,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193365994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193365995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193365995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193365995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:06,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193366014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T02:35:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031f3575b58cb046eb932c4e56f2f4a7b3_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f3575b58cb046eb932c4e56f2f4a7b3_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:06,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:06,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:06,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:06,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:06,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:06,032 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/422a41a308a9467bbb4a5ae2285a17df, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:06,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/422a41a308a9467bbb4a5ae2285a17df is 175, key is test_row_0/A:col10/1733193305590/Put/seqid=0 2024-12-03T02:35:06,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742189_1365 (size=39749) 2024-12-03T02:35:06,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/422a41a308a9467bbb4a5ae2285a17df 2024-12-03T02:35:06,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/78db62060d514464972544e5396e8e2f is 50, key is test_row_0/B:col10/1733193305590/Put/seqid=0 2024-12-03T02:35:06,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742190_1366 (size=12151) 2024-12-03T02:35:06,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/78db62060d514464972544e5396e8e2f 2024-12-03T02:35:06,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/1fa99f16295f432e860b7dbd1f45d633 is 50, key is test_row_0/C:col10/1733193305590/Put/seqid=0 2024-12-03T02:35:06,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742191_1367 (size=12151) 2024-12-03T02:35:06,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/1fa99f16295f432e860b7dbd1f45d633 2024-12-03T02:35:06,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/422a41a308a9467bbb4a5ae2285a17df as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/422a41a308a9467bbb4a5ae2285a17df 2024-12-03T02:35:06,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/422a41a308a9467bbb4a5ae2285a17df, entries=200, sequenceid=159, filesize=38.8 K 2024-12-03T02:35:06,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/78db62060d514464972544e5396e8e2f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/78db62060d514464972544e5396e8e2f 2024-12-03T02:35:06,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/78db62060d514464972544e5396e8e2f, entries=150, sequenceid=159, filesize=11.9 K 2024-12-03T02:35:06,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/1fa99f16295f432e860b7dbd1f45d633 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1fa99f16295f432e860b7dbd1f45d633 2024-12-03T02:35:06,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1fa99f16295f432e860b7dbd1f45d633, entries=150, sequenceid=159, filesize=11.9 K 2024-12-03T02:35:06,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 309d0efd241d9334329aa843e6c8f6f7 in 569ms, sequenceid=159, compaction requested=true 2024-12-03T02:35:06,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:06,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:06,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:06,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:06,161 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:06,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:06,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:06,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:35:06,162 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:06,162 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:06,162 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/A is initiating minor compaction (all files) 2024-12-03T02:35:06,162 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/A in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:06,162 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a91945940b4f42cabb0b5468afbfa69d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/75e429ce07224e508c78d3da93dec440, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/422a41a308a9467bbb4a5ae2285a17df] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=99.7 K 2024-12-03T02:35:06,162 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:06,162 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a91945940b4f42cabb0b5468afbfa69d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/75e429ce07224e508c78d3da93dec440, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/422a41a308a9467bbb4a5ae2285a17df] 2024-12-03T02:35:06,162 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:06,163 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/B is initiating minor compaction (all files) 2024-12-03T02:35:06,163 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/B in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:06,163 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5d5677613cc14ac29fd066f168bd43db, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/bc4ccd357e674a5d89dcd52a735593a1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/78db62060d514464972544e5396e8e2f] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=35.8 K 2024-12-03T02:35:06,163 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting a91945940b4f42cabb0b5468afbfa69d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733193302631 2024-12-03T02:35:06,163 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d5677613cc14ac29fd066f168bd43db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733193302631 2024-12-03T02:35:06,163 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e429ce07224e508c78d3da93dec440, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733193302681 2024-12-03T02:35:06,163 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc4ccd357e674a5d89dcd52a735593a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733193302681 2024-12-03T02:35:06,163 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 422a41a308a9467bbb4a5ae2285a17df, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733193304922 2024-12-03T02:35:06,164 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78db62060d514464972544e5396e8e2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733193304922 2024-12-03T02:35:06,171 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#B#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:06,171 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/daf5b23655754919a7b2aba9ea83fd94 is 50, key is test_row_0/B:col10/1733193305590/Put/seqid=0 2024-12-03T02:35:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T02:35:06,176 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:06,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742192_1368 (size=12561) 2024-12-03T02:35:06,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T02:35:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:06,187 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T02:35:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:06,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:06,200 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/daf5b23655754919a7b2aba9ea83fd94 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/daf5b23655754919a7b2aba9ea83fd94 2024-12-03T02:35:06,203 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203ad57ec923912434ab7715a92d9ed5a08_309d0efd241d9334329aa843e6c8f6f7 store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:06,204 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203ad57ec923912434ab7715a92d9ed5a08_309d0efd241d9334329aa843e6c8f6f7, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:06,205 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ad57ec923912434ab7715a92d9ed5a08_309d0efd241d9334329aa843e6c8f6f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:06,205 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/B of 309d0efd241d9334329aa843e6c8f6f7 into daf5b23655754919a7b2aba9ea83fd94(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:06,205 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:06,205 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/B, priority=13, startTime=1733193306161; duration=0sec 2024-12-03T02:35:06,205 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:06,205 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:B 2024-12-03T02:35:06,206 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:06,206 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:06,206 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/C is initiating minor compaction (all files) 2024-12-03T02:35:06,206 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/C in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:06,207 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/3fbd5ead21684506844d75d141c15029, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/519e2f6a4c7d474b8bdc3a1a5079ea8a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1fa99f16295f432e860b7dbd1f45d633] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=35.7 K 2024-12-03T02:35:06,207 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fbd5ead21684506844d75d141c15029, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733193302631 2024-12-03T02:35:06,207 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 519e2f6a4c7d474b8bdc3a1a5079ea8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733193302681 2024-12-03T02:35:06,208 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa99f16295f432e860b7dbd1f45d633, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733193304922 2024-12-03T02:35:06,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba5abc5adaf04fb28a1a2baadc770eb5_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193305661/Put/seqid=0 2024-12-03T02:35:06,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742193_1369 (size=4469) 2024-12-03T02:35:06,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742194_1370 (size=12304) 2024-12-03T02:35:06,229 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:06,230 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/68de51122dfc4b3585d74b8708145392 is 50, key is test_row_0/C:col10/1733193305590/Put/seqid=0 2024-12-03T02:35:06,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742195_1371 (size=12527) 2024-12-03T02:35:06,243 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/68de51122dfc4b3585d74b8708145392 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/68de51122dfc4b3585d74b8708145392 2024-12-03T02:35:06,248 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/C of 309d0efd241d9334329aa843e6c8f6f7 into 68de51122dfc4b3585d74b8708145392(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:06,248 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:06,248 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/C, priority=13, startTime=1733193306161; duration=0sec 2024-12-03T02:35:06,248 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:06,248 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:C 2024-12-03T02:35:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:06,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:06,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193366327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193366328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193366336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193366343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193366344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193366448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193366462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193366466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193366466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,612 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#A#compaction#306 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:06,613 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9272ab09e84e4c95885e6337df689473 is 175, key is test_row_0/A:col10/1733193305590/Put/seqid=0 2024-12-03T02:35:06,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742196_1372 (size=31515) 2024-12-03T02:35:06,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:06,639 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba5abc5adaf04fb28a1a2baadc770eb5_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba5abc5adaf04fb28a1a2baadc770eb5_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:06,640 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/9272ab09e84e4c95885e6337df689473 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9272ab09e84e4c95885e6337df689473 2024-12-03T02:35:06,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a95335516b9246a988c6cf1ecd2a17d0, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:06,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a95335516b9246a988c6cf1ecd2a17d0 is 175, key is test_row_0/A:col10/1733193305661/Put/seqid=0 2024-12-03T02:35:06,647 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/A of 309d0efd241d9334329aa843e6c8f6f7 into 9272ab09e84e4c95885e6337df689473(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:06,647 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:06,647 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/A, priority=13, startTime=1733193306161; duration=0sec 2024-12-03T02:35:06,648 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:06,649 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:A 2024-12-03T02:35:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T02:35:06,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193366667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742197_1373 (size=31105) 2024-12-03T02:35:06,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193366695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193366696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193366697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193366854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:07,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193366995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:07,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193367016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:07,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193367018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:07,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193367019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:07,099 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a95335516b9246a988c6cf1ecd2a17d0 2024-12-03T02:35:07,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/13bce31cd25f4967a401e3215432cad4 is 50, key is test_row_0/B:col10/1733193305661/Put/seqid=0 2024-12-03T02:35:07,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742198_1374 (size=12151) 2024-12-03T02:35:07,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193367514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:07,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:07,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193367534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193367534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:07,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:07,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193367545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:07,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:07,573 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/13bce31cd25f4967a401e3215432cad4 2024-12-03T02:35:07,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/eda3ea3d721b45bb8937a82a2da86947 is 50, key is test_row_0/C:col10/1733193305661/Put/seqid=0 2024-12-03T02:35:07,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742199_1375 (size=12151) 2024-12-03T02:35:07,632 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/eda3ea3d721b45bb8937a82a2da86947 2024-12-03T02:35:07,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a95335516b9246a988c6cf1ecd2a17d0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a95335516b9246a988c6cf1ecd2a17d0 2024-12-03T02:35:07,644 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a95335516b9246a988c6cf1ecd2a17d0, entries=150, sequenceid=178, filesize=30.4 K 2024-12-03T02:35:07,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/13bce31cd25f4967a401e3215432cad4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/13bce31cd25f4967a401e3215432cad4 2024-12-03T02:35:07,656 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/13bce31cd25f4967a401e3215432cad4, entries=150, sequenceid=178, filesize=11.9 K 2024-12-03T02:35:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/eda3ea3d721b45bb8937a82a2da86947 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/eda3ea3d721b45bb8937a82a2da86947 2024-12-03T02:35:07,670 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/eda3ea3d721b45bb8937a82a2da86947, entries=150, sequenceid=178, filesize=11.9 K 2024-12-03T02:35:07,671 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 309d0efd241d9334329aa843e6c8f6f7 in 1484ms, sequenceid=178, compaction requested=false 2024-12-03T02:35:07,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:07,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:07,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-03T02:35:07,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-03T02:35:07,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-03T02:35:07,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1380 sec 2024-12-03T02:35:07,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.1520 sec 2024-12-03T02:35:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T02:35:07,695 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:07,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-03T02:35:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T02:35:07,698 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:07,699 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:07,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T02:35:07,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T02:35:07,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:07,860 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T02:35:07,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:07,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:07,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:07,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:07,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:07,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:07,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:07,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035587d91a4aee459db55bfa97eb0d70d9_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193306335/Put/seqid=0 2024-12-03T02:35:07,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742200_1376 (size=12304) 2024-12-03T02:35:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:07,914 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035587d91a4aee459db55bfa97eb0d70d9_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035587d91a4aee459db55bfa97eb0d70d9_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/ded6a2abb9fd43f790ebccc974e7849e, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/ded6a2abb9fd43f790ebccc974e7849e is 175, key is test_row_0/A:col10/1733193306335/Put/seqid=0 2024-12-03T02:35:07,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742201_1377 (size=31105) 2024-12-03T02:35:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T02:35:08,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193368033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:08,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193368154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:08,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T02:35:08,342 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/ded6a2abb9fd43f790ebccc974e7849e 2024-12-03T02:35:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/b607f1d522dc4fc3a30a7bcb89cbda46 is 50, key is test_row_0/B:col10/1733193306335/Put/seqid=0 2024-12-03T02:35:08,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193368374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:08,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742202_1378 (size=12151) 2024-12-03T02:35:08,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193368538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:08,548 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 309d0efd241d9334329aa843e6c8f6f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:08,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193368556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:08,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193368556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:08,562 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in 309d0efd241d9334329aa843e6c8f6f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,562 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 309d0efd241d9334329aa843e6c8f6f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:08,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193368565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:08,573 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 309d0efd241d9334329aa843e6c8f6f7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:08,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193368695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:08,796 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/b607f1d522dc4fc3a30a7bcb89cbda46 2024-12-03T02:35:08,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/563f0c36bdb542b19cb2252f52884e9d is 50, key is test_row_0/C:col10/1733193306335/Put/seqid=0 2024-12-03T02:35:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T02:35:08,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742203_1379 (size=12151) 2024-12-03T02:35:08,864 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/563f0c36bdb542b19cb2252f52884e9d 2024-12-03T02:35:08,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/ded6a2abb9fd43f790ebccc974e7849e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/ded6a2abb9fd43f790ebccc974e7849e 2024-12-03T02:35:08,872 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/ded6a2abb9fd43f790ebccc974e7849e, entries=150, sequenceid=194, filesize=30.4 K 2024-12-03T02:35:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/b607f1d522dc4fc3a30a7bcb89cbda46 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/b607f1d522dc4fc3a30a7bcb89cbda46 2024-12-03T02:35:08,884 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/b607f1d522dc4fc3a30a7bcb89cbda46, entries=150, sequenceid=194, filesize=11.9 K 2024-12-03T02:35:08,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/563f0c36bdb542b19cb2252f52884e9d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/563f0c36bdb542b19cb2252f52884e9d 2024-12-03T02:35:08,895 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/563f0c36bdb542b19cb2252f52884e9d, entries=150, sequenceid=194, filesize=11.9 K 2024-12-03T02:35:08,900 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 309d0efd241d9334329aa843e6c8f6f7 in 1038ms, sequenceid=194, compaction requested=true 2024-12-03T02:35:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-03T02:35:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-03T02:35:08,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-03T02:35:08,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2030 sec 2024-12-03T02:35:08,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.2090 sec 2024-12-03T02:35:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:09,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T02:35:09,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:09,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:09,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:09,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:09,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:09,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:09,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039664b269e765436badacb35113a287d6_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193308029/Put/seqid=0 2024-12-03T02:35:09,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742204_1380 (size=14794) 2024-12-03T02:35:09,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193369309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:09,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:09,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193369427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:09,636 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:09,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039664b269e765436badacb35113a287d6_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039664b269e765436badacb35113a287d6_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:09,645 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a742b95dc70f404d82c691311dffa675, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:09,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a742b95dc70f404d82c691311dffa675 is 175, key is test_row_0/A:col10/1733193308029/Put/seqid=0 2024-12-03T02:35:09,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193369639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:09,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742205_1381 (size=39749) 2024-12-03T02:35:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T02:35:09,845 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:09,845 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-03T02:35:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T02:35:09,847 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:09,847 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:09,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:09,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T02:35:09,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193369965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:09,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T02:35:09,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:09,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:09,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:09,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:09,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:10,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a742b95dc70f404d82c691311dffa675 2024-12-03T02:35:10,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/c080b445bd0143ce80b835502b260b88 is 50, key is test_row_0/B:col10/1733193308029/Put/seqid=0 2024-12-03T02:35:10,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742206_1382 (size=12151) 2024-12-03T02:35:10,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/c080b445bd0143ce80b835502b260b88 2024-12-03T02:35:10,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/8ea951353f1a446a84e94c4e4e792df6 is 50, key is test_row_0/C:col10/1733193308029/Put/seqid=0 2024-12-03T02:35:10,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742207_1383 (size=12151) 2024-12-03T02:35:10,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/8ea951353f1a446a84e94c4e4e792df6 2024-12-03T02:35:10,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/a742b95dc70f404d82c691311dffa675 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a742b95dc70f404d82c691311dffa675 2024-12-03T02:35:10,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T02:35:10,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:10,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:10,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:10,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:10,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:10,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a742b95dc70f404d82c691311dffa675, entries=200, sequenceid=214, filesize=38.8 K 2024-12-03T02:35:10,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/c080b445bd0143ce80b835502b260b88 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c080b445bd0143ce80b835502b260b88 2024-12-03T02:35:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T02:35:10,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c080b445bd0143ce80b835502b260b88, entries=150, sequenceid=214, filesize=11.9 K 2024-12-03T02:35:10,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/8ea951353f1a446a84e94c4e4e792df6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8ea951353f1a446a84e94c4e4e792df6 2024-12-03T02:35:10,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8ea951353f1a446a84e94c4e4e792df6, entries=150, sequenceid=214, filesize=11.9 K 2024-12-03T02:35:10,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 309d0efd241d9334329aa843e6c8f6f7 in 957ms, sequenceid=214, compaction requested=true 2024-12-03T02:35:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:10,178 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:10,178 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:10,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:10,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:10,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:10,181 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:10,181 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/A is initiating minor compaction (all files) 2024-12-03T02:35:10,181 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:10,181 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/B is initiating minor compaction (all files) 2024-12-03T02:35:10,181 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/A in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:10,181 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/B in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:10,181 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/daf5b23655754919a7b2aba9ea83fd94, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/13bce31cd25f4967a401e3215432cad4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/b607f1d522dc4fc3a30a7bcb89cbda46, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c080b445bd0143ce80b835502b260b88] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=47.9 K 2024-12-03T02:35:10,181 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9272ab09e84e4c95885e6337df689473, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a95335516b9246a988c6cf1ecd2a17d0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/ded6a2abb9fd43f790ebccc974e7849e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a742b95dc70f404d82c691311dffa675] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=130.3 K 2024-12-03T02:35:10,181 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:10,181 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9272ab09e84e4c95885e6337df689473, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a95335516b9246a988c6cf1ecd2a17d0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/ded6a2abb9fd43f790ebccc974e7849e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a742b95dc70f404d82c691311dffa675] 2024-12-03T02:35:10,181 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting daf5b23655754919a7b2aba9ea83fd94, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733193304922 2024-12-03T02:35:10,181 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9272ab09e84e4c95885e6337df689473, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733193304922 2024-12-03T02:35:10,182 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 13bce31cd25f4967a401e3215432cad4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733193305645 2024-12-03T02:35:10,182 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a95335516b9246a988c6cf1ecd2a17d0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733193305645 2024-12-03T02:35:10,182 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b607f1d522dc4fc3a30a7bcb89cbda46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733193306325 2024-12-03T02:35:10,182 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded6a2abb9fd43f790ebccc974e7849e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733193306325 2024-12-03T02:35:10,182 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting c080b445bd0143ce80b835502b260b88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733193307962 2024-12-03T02:35:10,182 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a742b95dc70f404d82c691311dffa675, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733193307962 2024-12-03T02:35:10,203 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:10,203 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f78b217bf17042c590ff343f9ff426d1 is 50, key is test_row_0/B:col10/1733193308029/Put/seqid=0 2024-12-03T02:35:10,220 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:10,227 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203811618610f61487f842c15e4499fa000_309d0efd241d9334329aa843e6c8f6f7 store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:10,229 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203811618610f61487f842c15e4499fa000_309d0efd241d9334329aa843e6c8f6f7, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:10,229 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203811618610f61487f842c15e4499fa000_309d0efd241d9334329aa843e6c8f6f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:10,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742208_1384 (size=12697) 2024-12-03T02:35:10,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742209_1385 (size=4469) 2024-12-03T02:35:10,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T02:35:10,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:10,323 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T02:35:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:10,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120328fdc157d2644b6a86a12b1c5f94be55_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193309307/Put/seqid=0 2024-12-03T02:35:10,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742210_1386 (size=12304) 2024-12-03T02:35:10,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:10,414 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120328fdc157d2644b6a86a12b1c5f94be55_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120328fdc157d2644b6a86a12b1c5f94be55_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:10,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/bc46db65e08748d2b090e7054e478f6f, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:10,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/bc46db65e08748d2b090e7054e478f6f is 175, key is test_row_0/A:col10/1733193309307/Put/seqid=0 2024-12-03T02:35:10,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742211_1387 (size=31105) 2024-12-03T02:35:10,458 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/bc46db65e08748d2b090e7054e478f6f 2024-12-03T02:35:10,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/9540531a244a47768a035b8aeff2c51c is 50, key is test_row_0/B:col10/1733193309307/Put/seqid=0 2024-12-03T02:35:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T02:35:10,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:10,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742212_1388 (size=12151) 2024-12-03T02:35:10,520 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/9540531a244a47768a035b8aeff2c51c 2024-12-03T02:35:10,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/0fa222f466d5486d97ff9adc38760aea is 50, key is test_row_0/C:col10/1733193309307/Put/seqid=0 2024-12-03T02:35:10,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742213_1389 (size=12151) 2024-12-03T02:35:10,554 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/0fa222f466d5486d97ff9adc38760aea 2024-12-03T02:35:10,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/bc46db65e08748d2b090e7054e478f6f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/bc46db65e08748d2b090e7054e478f6f 2024-12-03T02:35:10,564 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/bc46db65e08748d2b090e7054e478f6f, entries=150, sequenceid=226, filesize=30.4 K 2024-12-03T02:35:10,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/9540531a244a47768a035b8aeff2c51c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/9540531a244a47768a035b8aeff2c51c 2024-12-03T02:35:10,569 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/9540531a244a47768a035b8aeff2c51c, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T02:35:10,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/0fa222f466d5486d97ff9adc38760aea as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0fa222f466d5486d97ff9adc38760aea 2024-12-03T02:35:10,581 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0fa222f466d5486d97ff9adc38760aea, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T02:35:10,587 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=73.80 KB/75570 for 309d0efd241d9334329aa843e6c8f6f7 in 264ms, sequenceid=226, compaction requested=true 2024-12-03T02:35:10,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:10,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:10,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-03T02:35:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-03T02:35:10,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T02:35:10,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-03T02:35:10,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 742 msec 2024-12-03T02:35:10,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 748 msec 2024-12-03T02:35:10,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:10,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:10,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:10,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:10,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:10,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:10,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f9099b30afdf4b11bceb25bfdceea297_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193310559/Put/seqid=0 2024-12-03T02:35:10,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193370623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193370636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193370638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193370638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193370638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742214_1390 (size=14794) 2024-12-03T02:35:10,657 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:10,664 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/f78b217bf17042c590ff343f9ff426d1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f78b217bf17042c590ff343f9ff426d1 2024-12-03T02:35:10,668 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/B of 309d0efd241d9334329aa843e6c8f6f7 into f78b217bf17042c590ff343f9ff426d1(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:10,668 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:10,668 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/B, priority=12, startTime=1733193310178; duration=0sec 2024-12-03T02:35:10,668 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:10,668 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:B 2024-12-03T02:35:10,669 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T02:35:10,671 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T02:35:10,671 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/C is initiating minor compaction (all files) 2024-12-03T02:35:10,671 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/C in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:10,671 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/68de51122dfc4b3585d74b8708145392, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/eda3ea3d721b45bb8937a82a2da86947, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/563f0c36bdb542b19cb2252f52884e9d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8ea951353f1a446a84e94c4e4e792df6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0fa222f466d5486d97ff9adc38760aea] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=59.7 K 2024-12-03T02:35:10,671 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f9099b30afdf4b11bceb25bfdceea297_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f9099b30afdf4b11bceb25bfdceea297_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:10,672 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/64a04f0f72c14a96b7eeafa9c7ae1f73, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:10,672 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 68de51122dfc4b3585d74b8708145392, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733193304922 2024-12-03T02:35:10,673 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting eda3ea3d721b45bb8937a82a2da86947, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733193305645 2024-12-03T02:35:10,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/64a04f0f72c14a96b7eeafa9c7ae1f73 is 175, key is test_row_0/A:col10/1733193310559/Put/seqid=0 2024-12-03T02:35:10,674 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 563f0c36bdb542b19cb2252f52884e9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733193306325 2024-12-03T02:35:10,675 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea951353f1a446a84e94c4e4e792df6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733193307962 2024-12-03T02:35:10,678 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fa222f466d5486d97ff9adc38760aea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733193309240 2024-12-03T02:35:10,693 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#A#compaction#318 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:10,693 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/035f7213ec414374952547c360ac7c1d is 175, key is test_row_0/A:col10/1733193308029/Put/seqid=0 2024-12-03T02:35:10,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742215_1391 (size=39749) 2024-12-03T02:35:10,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742216_1392 (size=31651) 2024-12-03T02:35:10,705 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/64a04f0f72c14a96b7eeafa9c7ae1f73 2024-12-03T02:35:10,710 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:10,711 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/f1a176284a2843c5a22b57cd631eaab1 is 50, key is test_row_0/C:col10/1733193309307/Put/seqid=0 2024-12-03T02:35:10,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/05d239cc27f04d61827cd12114b77737 is 50, key is test_row_0/B:col10/1733193310559/Put/seqid=0 2024-12-03T02:35:10,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193370749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193370749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742217_1393 (size=12697) 2024-12-03T02:35:10,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193370772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193370772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193370775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742218_1394 (size=12151) 2024-12-03T02:35:10,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/05d239cc27f04d61827cd12114b77737 2024-12-03T02:35:10,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/d6122f2a609c472abe6dd61bd92809c7 is 50, key is test_row_0/C:col10/1733193310559/Put/seqid=0 2024-12-03T02:35:10,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742219_1395 (size=12151) 2024-12-03T02:35:10,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193370965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193370967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T02:35:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:10,985 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:10,985 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-03T02:35:10,988 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:10,988 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:10,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T02:35:11,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193370996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193370996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193370999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T02:35:11,136 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/035f7213ec414374952547c360ac7c1d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/035f7213ec414374952547c360ac7c1d 2024-12-03T02:35:11,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T02:35:11,148 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/A of 309d0efd241d9334329aa843e6c8f6f7 into 035f7213ec414374952547c360ac7c1d(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:11,148 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:11,148 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/A, priority=12, startTime=1733193310178; duration=0sec 2024-12-03T02:35:11,148 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:11,148 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:A 2024-12-03T02:35:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:11,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:11,190 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/f1a176284a2843c5a22b57cd631eaab1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/f1a176284a2843c5a22b57cd631eaab1 2024-12-03T02:35:11,212 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/C of 309d0efd241d9334329aa843e6c8f6f7 into f1a176284a2843c5a22b57cd631eaab1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:11,213 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:11,213 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/C, priority=11, startTime=1733193310178; duration=0sec 2024-12-03T02:35:11,213 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:11,213 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:C 2024-12-03T02:35:11,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/d6122f2a609c472abe6dd61bd92809c7 2024-12-03T02:35:11,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T02:35:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:11,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:11,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193371295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T02:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193371295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/64a04f0f72c14a96b7eeafa9c7ae1f73 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/64a04f0f72c14a96b7eeafa9c7ae1f73 2024-12-03T02:35:11,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193371325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193371326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193371326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/64a04f0f72c14a96b7eeafa9c7ae1f73, entries=200, sequenceid=242, filesize=38.8 K 2024-12-03T02:35:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/05d239cc27f04d61827cd12114b77737 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/05d239cc27f04d61827cd12114b77737 2024-12-03T02:35:11,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/05d239cc27f04d61827cd12114b77737, entries=150, sequenceid=242, filesize=11.9 K 2024-12-03T02:35:11,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/d6122f2a609c472abe6dd61bd92809c7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/d6122f2a609c472abe6dd61bd92809c7 2024-12-03T02:35:11,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/d6122f2a609c472abe6dd61bd92809c7, entries=150, sequenceid=242, filesize=11.9 K 2024-12-03T02:35:11,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 309d0efd241d9334329aa843e6c8f6f7 in 788ms, sequenceid=242, compaction requested=true 2024-12-03T02:35:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:11,380 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:11,380 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:11,381 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/A is initiating minor compaction (all files) 2024-12-03T02:35:11,381 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/A in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:11,381 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/035f7213ec414374952547c360ac7c1d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/bc46db65e08748d2b090e7054e478f6f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/64a04f0f72c14a96b7eeafa9c7ae1f73] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=100.1 K 2024-12-03T02:35:11,381 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:11,381 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/035f7213ec414374952547c360ac7c1d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/bc46db65e08748d2b090e7054e478f6f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/64a04f0f72c14a96b7eeafa9c7ae1f73] 2024-12-03T02:35:11,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:11,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:11,381 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:11,381 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 035f7213ec414374952547c360ac7c1d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733193307962 2024-12-03T02:35:11,382 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc46db65e08748d2b090e7054e478f6f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733193309240 2024-12-03T02:35:11,382 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64a04f0f72c14a96b7eeafa9c7ae1f73, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733193310559 2024-12-03T02:35:11,382 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:11,382 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/B is initiating minor compaction (all files) 2024-12-03T02:35:11,382 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/B in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:11,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:11,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:11,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:11,382 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f78b217bf17042c590ff343f9ff426d1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/9540531a244a47768a035b8aeff2c51c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/05d239cc27f04d61827cd12114b77737] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=36.1 K 2024-12-03T02:35:11,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:11,383 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting f78b217bf17042c590ff343f9ff426d1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733193307962 2024-12-03T02:35:11,383 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9540531a244a47768a035b8aeff2c51c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733193309240 2024-12-03T02:35:11,383 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d239cc27f04d61827cd12114b77737, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733193310559 2024-12-03T02:35:11,396 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:11,399 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:11,399 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/917a30799a8b4658b625db1aa390eaa9 is 50, key is test_row_0/B:col10/1733193310559/Put/seqid=0 2024-12-03T02:35:11,408 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f2e54323949e4829893430d1d7e995bd_309d0efd241d9334329aa843e6c8f6f7 store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:11,409 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f2e54323949e4829893430d1d7e995bd_309d0efd241d9334329aa843e6c8f6f7, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:11,410 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f2e54323949e4829893430d1d7e995bd_309d0efd241d9334329aa843e6c8f6f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:11,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742220_1396 (size=12799) 2024-12-03T02:35:11,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742221_1397 (size=4469) 2024-12-03T02:35:11,450 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#A#compaction#326 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:11,451 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/cda861360458442ea683932b7d8769b3 is 175, key is test_row_0/A:col10/1733193310559/Put/seqid=0 2024-12-03T02:35:11,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T02:35:11,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:11,464 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T02:35:11,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:11,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742222_1398 (size=31753) 2024-12-03T02:35:11,503 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/cda861360458442ea683932b7d8769b3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/cda861360458442ea683932b7d8769b3 2024-12-03T02:35:11,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203da0620180e0a4a719001ae0718fb1657_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193310637/Put/seqid=0 2024-12-03T02:35:11,516 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/A of 309d0efd241d9334329aa843e6c8f6f7 into cda861360458442ea683932b7d8769b3(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:11,516 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:11,516 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/A, priority=13, startTime=1733193311380; duration=0sec 2024-12-03T02:35:11,517 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:11,517 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:A 2024-12-03T02:35:11,518 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T02:35:11,520 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T02:35:11,520 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T02:35:11,520 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. because compaction request was cancelled 2024-12-03T02:35:11,520 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:C 2024-12-03T02:35:11,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742223_1399 (size=12354) 2024-12-03T02:35:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T02:35:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:11,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:11,890 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/917a30799a8b4658b625db1aa390eaa9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/917a30799a8b4658b625db1aa390eaa9 2024-12-03T02:35:11,899 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/B of 309d0efd241d9334329aa843e6c8f6f7 into 917a30799a8b4658b625db1aa390eaa9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:11,899 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:11,899 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/B, priority=13, startTime=1733193311381; duration=0sec 2024-12-03T02:35:11,899 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:11,899 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:B 2024-12-03T02:35:11,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193371876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193371877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193371887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193371893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193371893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:11,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:11,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:11,943 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203da0620180e0a4a719001ae0718fb1657_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203da0620180e0a4a719001ae0718fb1657_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:11,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/e7771c5c39264fa0925790d4b23e06d2, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/e7771c5c39264fa0925790d4b23e06d2 is 175, key is test_row_0/A:col10/1733193310637/Put/seqid=0 2024-12-03T02:35:12,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742224_1400 (size=31155) 2024-12-03T02:35:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193372015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193372015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193372015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193372015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193372015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T02:35:12,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193372236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193372236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193372236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193372236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193372236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,410 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/e7771c5c39264fa0925790d4b23e06d2 2024-12-03T02:35:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/989b3c55e24e40fda19fda21de075a6a is 50, key is test_row_0/B:col10/1733193310637/Put/seqid=0 2024-12-03T02:35:12,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742225_1401 (size=12201) 2024-12-03T02:35:12,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193372545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193372546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193372546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193372547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193372555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:12,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:12,851 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/989b3c55e24e40fda19fda21de075a6a 2024-12-03T02:35:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/1a9b7bc25f2c446d9c1c93f77ca98f96 is 50, key is test_row_0/C:col10/1733193310637/Put/seqid=0 2024-12-03T02:35:12,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742226_1402 (size=12201) 2024-12-03T02:35:12,867 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/1a9b7bc25f2c446d9c1c93f77ca98f96 2024-12-03T02:35:12,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/e7771c5c39264fa0925790d4b23e06d2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/e7771c5c39264fa0925790d4b23e06d2 2024-12-03T02:35:12,882 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/e7771c5c39264fa0925790d4b23e06d2, entries=150, sequenceid=261, filesize=30.4 K 2024-12-03T02:35:12,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/989b3c55e24e40fda19fda21de075a6a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/989b3c55e24e40fda19fda21de075a6a 2024-12-03T02:35:12,887 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/989b3c55e24e40fda19fda21de075a6a, entries=150, sequenceid=261, filesize=11.9 K 2024-12-03T02:35:12,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/1a9b7bc25f2c446d9c1c93f77ca98f96 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1a9b7bc25f2c446d9c1c93f77ca98f96 2024-12-03T02:35:12,891 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1a9b7bc25f2c446d9c1c93f77ca98f96, entries=150, sequenceid=261, filesize=11.9 K 2024-12-03T02:35:12,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-03T02:35:12,892 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 309d0efd241d9334329aa843e6c8f6f7 in 1428ms, sequenceid=261, compaction requested=true 2024-12-03T02:35:12,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:12,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:12,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-03T02:35:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-03T02:35:12,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-03T02:35:12,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9040 sec 2024-12-03T02:35:12,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.9130 sec 2024-12-03T02:35:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:13,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T02:35:13,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:13,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:13,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:13,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:13,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:13,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:13,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120315e4ed8b07b14869bfe83aaff79c2a5c_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193311887/Put/seqid=0 2024-12-03T02:35:13,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742227_1403 (size=14994) 2024-12-03T02:35:13,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:13,087 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120315e4ed8b07b14869bfe83aaff79c2a5c_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120315e4ed8b07b14869bfe83aaff79c2a5c_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:13,088 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/25403dc2901642689e5bc989046b9272, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:13,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/25403dc2901642689e5bc989046b9272 is 175, key is test_row_0/A:col10/1733193311887/Put/seqid=0 2024-12-03T02:35:13,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742228_1404 (size=39949) 2024-12-03T02:35:13,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193373109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193373109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193373109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193373110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193373116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T02:35:13,146 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:13,147 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-03T02:35:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T02:35:13,149 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:13,152 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:13,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:13,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193373235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193373236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193373236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193373236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193373240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T02:35:13,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T02:35:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193373445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193373447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193373455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193373455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193373455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T02:35:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T02:35:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,494 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/25403dc2901642689e5bc989046b9272 2024-12-03T02:35:13,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/efb5b59e5fb542d0998fc98252f7b62a is 50, key is test_row_0/B:col10/1733193311887/Put/seqid=0 2024-12-03T02:35:13,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742229_1405 (size=12301) 2024-12-03T02:35:13,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T02:35:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193373754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193373764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193373764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193373764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T02:35:13,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:13,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T02:35:13,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193373775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:13,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/efb5b59e5fb542d0998fc98252f7b62a 2024-12-03T02:35:13,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T02:35:13,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:13,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:13,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/921e145c4181412bbefb2ad5e5389e62 is 50, key is test_row_0/C:col10/1733193311887/Put/seqid=0 2024-12-03T02:35:13,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742230_1406 (size=12301) 2024-12-03T02:35:13,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/921e145c4181412bbefb2ad5e5389e62 2024-12-03T02:35:13,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/25403dc2901642689e5bc989046b9272 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25403dc2901642689e5bc989046b9272 2024-12-03T02:35:13,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25403dc2901642689e5bc989046b9272, entries=200, sequenceid=280, filesize=39.0 K 2024-12-03T02:35:13,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/efb5b59e5fb542d0998fc98252f7b62a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/efb5b59e5fb542d0998fc98252f7b62a 2024-12-03T02:35:13,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/efb5b59e5fb542d0998fc98252f7b62a, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T02:35:13,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/921e145c4181412bbefb2ad5e5389e62 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/921e145c4181412bbefb2ad5e5389e62 2024-12-03T02:35:13,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/921e145c4181412bbefb2ad5e5389e62, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T02:35:13,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 309d0efd241d9334329aa843e6c8f6f7 in 917ms, sequenceid=280, compaction requested=true 2024-12-03T02:35:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:13,986 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:13,986 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:13,991 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:13,991 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/B is initiating minor compaction (all files) 2024-12-03T02:35:13,991 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/B in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,992 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/917a30799a8b4658b625db1aa390eaa9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/989b3c55e24e40fda19fda21de075a6a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/efb5b59e5fb542d0998fc98252f7b62a] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=36.4 K 2024-12-03T02:35:13,992 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:13,992 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/A is initiating minor compaction (all files) 2024-12-03T02:35:13,992 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/A in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,992 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/cda861360458442ea683932b7d8769b3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/e7771c5c39264fa0925790d4b23e06d2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25403dc2901642689e5bc989046b9272] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=100.4 K 2024-12-03T02:35:13,992 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:13,992 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/cda861360458442ea683932b7d8769b3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/e7771c5c39264fa0925790d4b23e06d2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25403dc2901642689e5bc989046b9272] 2024-12-03T02:35:13,993 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 917a30799a8b4658b625db1aa390eaa9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733193310559 2024-12-03T02:35:13,993 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting cda861360458442ea683932b7d8769b3, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733193310559 2024-12-03T02:35:13,994 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 989b3c55e24e40fda19fda21de075a6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733193310634 2024-12-03T02:35:13,994 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7771c5c39264fa0925790d4b23e06d2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733193310634 2024-12-03T02:35:13,994 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting efb5b59e5fb542d0998fc98252f7b62a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733193311865 2024-12-03T02:35:13,995 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25403dc2901642689e5bc989046b9272, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733193311865 2024-12-03T02:35:14,012 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:14,015 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#B#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:14,016 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/57f6d151b237413185c699ec44fd6b2e is 50, key is test_row_0/B:col10/1733193311887/Put/seqid=0 2024-12-03T02:35:14,017 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203cc5268c60d6a470eaec92e09d6fb609d_309d0efd241d9334329aa843e6c8f6f7 store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:14,018 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203cc5268c60d6a470eaec92e09d6fb609d_309d0efd241d9334329aa843e6c8f6f7, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:14,019 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cc5268c60d6a470eaec92e09d6fb609d_309d0efd241d9334329aa843e6c8f6f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:14,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742231_1407 (size=13051) 2024-12-03T02:35:14,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742232_1408 (size=4469) 2024-12-03T02:35:14,023 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#A#compaction#334 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:14,024 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/70b3295c204044d19f3ed37e49456839 is 175, key is test_row_0/A:col10/1733193311887/Put/seqid=0 2024-12-03T02:35:14,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742233_1409 (size=32005) 2024-12-03T02:35:14,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T02:35:14,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:14,078 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T02:35:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:14,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120311cc87fd2b594cf2bd0816ec8ee38416_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193313110/Put/seqid=0 2024-12-03T02:35:14,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742234_1410 (size=12454) 2024-12-03T02:35:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:14,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T02:35:14,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193374307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193374307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193374317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193374317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193374318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193374425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193374425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193374426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,430 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/57f6d151b237413185c699ec44fd6b2e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/57f6d151b237413185c699ec44fd6b2e 2024-12-03T02:35:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,435 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/B of 309d0efd241d9334329aa843e6c8f6f7 into 57f6d151b237413185c699ec44fd6b2e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:14,435 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:14,435 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/B, priority=13, startTime=1733193313986; duration=0sec 2024-12-03T02:35:14,435 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:14,435 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:B 2024-12-03T02:35:14,435 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:14,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193374434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,442 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49350 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:14,442 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/C is initiating minor compaction (all files) 2024-12-03T02:35:14,442 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/C in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,442 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/f1a176284a2843c5a22b57cd631eaab1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/d6122f2a609c472abe6dd61bd92809c7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1a9b7bc25f2c446d9c1c93f77ca98f96, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/921e145c4181412bbefb2ad5e5389e62] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=48.2 K 2024-12-03T02:35:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193374434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,443 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting f1a176284a2843c5a22b57cd631eaab1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733193309240 2024-12-03T02:35:14,444 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d6122f2a609c472abe6dd61bd92809c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733193310559 2024-12-03T02:35:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,446 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/70b3295c204044d19f3ed37e49456839 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/70b3295c204044d19f3ed37e49456839 2024-12-03T02:35:14,446 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a9b7bc25f2c446d9c1c93f77ca98f96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733193310634 2024-12-03T02:35:14,447 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 921e145c4181412bbefb2ad5e5389e62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733193311865 2024-12-03T02:35:14,452 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/A of 309d0efd241d9334329aa843e6c8f6f7 into 70b3295c204044d19f3ed37e49456839(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:14,452 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:14,452 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/A, priority=13, startTime=1733193313986; duration=0sec 2024-12-03T02:35:14,452 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:14,452 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:A 2024-12-03T02:35:14,466 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#C#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:14,466 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/2ac422e8d9d44027ab75c44f20d4d35b is 50, key is test_row_0/C:col10/1733193311887/Put/seqid=0 2024-12-03T02:35:14,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742235_1411 (size=12983) 2024-12-03T02:35:14,486 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/2ac422e8d9d44027ab75c44f20d4d35b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/2ac422e8d9d44027ab75c44f20d4d35b 2024-12-03T02:35:14,495 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/C of 309d0efd241d9334329aa843e6c8f6f7 into 2ac422e8d9d44027ab75c44f20d4d35b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:14,495 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:14,495 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/C, priority=12, startTime=1733193313986; duration=0sec 2024-12-03T02:35:14,495 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:14,495 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:C 2024-12-03T02:35:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:14,510 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120311cc87fd2b594cf2bd0816ec8ee38416_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120311cc87fd2b594cf2bd0816ec8ee38416_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:14,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/94c27b2890544cbfb5f5aab1f5341ef9, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:14,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/94c27b2890544cbfb5f5aab1f5341ef9 is 175, key is test_row_0/A:col10/1733193313110/Put/seqid=0 2024-12-03T02:35:14,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742236_1412 (size=31255) 2024-12-03T02:35:14,536 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/94c27b2890544cbfb5f5aab1f5341ef9 2024-12-03T02:35:14,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5617b43336c746239fa9549f1bd75cce is 50, key is test_row_0/B:col10/1733193313110/Put/seqid=0 2024-12-03T02:35:14,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742237_1413 (size=12301) 2024-12-03T02:35:14,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193374634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193374634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193374634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193374644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193374655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193374945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193374955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193374955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193374955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193374968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:14,981 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5617b43336c746239fa9549f1bd75cce 2024-12-03T02:35:15,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/43ebfb21eec845e6a23230b7c2a3fad8 is 50, key is test_row_0/C:col10/1733193313110/Put/seqid=0 2024-12-03T02:35:15,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742238_1414 (size=12301) 2024-12-03T02:35:15,043 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/43ebfb21eec845e6a23230b7c2a3fad8 2024-12-03T02:35:15,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/94c27b2890544cbfb5f5aab1f5341ef9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/94c27b2890544cbfb5f5aab1f5341ef9 2024-12-03T02:35:15,088 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/94c27b2890544cbfb5f5aab1f5341ef9, entries=150, sequenceid=295, filesize=30.5 K 2024-12-03T02:35:15,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5617b43336c746239fa9549f1bd75cce as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5617b43336c746239fa9549f1bd75cce 2024-12-03T02:35:15,127 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5617b43336c746239fa9549f1bd75cce, entries=150, sequenceid=295, filesize=12.0 K 2024-12-03T02:35:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/43ebfb21eec845e6a23230b7c2a3fad8 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/43ebfb21eec845e6a23230b7c2a3fad8 2024-12-03T02:35:15,142 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/43ebfb21eec845e6a23230b7c2a3fad8, entries=150, sequenceid=295, filesize=12.0 K 2024-12-03T02:35:15,143 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 309d0efd241d9334329aa843e6c8f6f7 in 1065ms, sequenceid=295, compaction requested=false 2024-12-03T02:35:15,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:15,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:15,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-03T02:35:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-03T02:35:15,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-03T02:35:15,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9940 sec 2024-12-03T02:35:15,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0030 sec 2024-12-03T02:35:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T02:35:15,305 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:15,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-03T02:35:15,308 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T02:35:15,311 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:15,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T02:35:15,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T02:35:15,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:15,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:15,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:15,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:15,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:15,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:15,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T02:35:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:15,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:15,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:15,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ef076868706144cb8d3c550aa624e2a8_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193314317/Put/seqid=0 2024-12-03T02:35:15,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193375497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193375505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193375520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193375520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193375521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742239_1415 (size=12454) 2024-12-03T02:35:15,567 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:15,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ef076868706144cb8d3c550aa624e2a8_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ef076868706144cb8d3c550aa624e2a8_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:15,585 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/25fe5a3392084f1aa308a329e35c73bf, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:15,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/25fe5a3392084f1aa308a329e35c73bf is 175, key is test_row_0/A:col10/1733193314317/Put/seqid=0 2024-12-03T02:35:15,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742240_1416 (size=31255) 2024-12-03T02:35:15,592 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/25fe5a3392084f1aa308a329e35c73bf 2024-12-03T02:35:15,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/4a504e078dac445a9a0429c90db5288d is 50, key is test_row_0/B:col10/1733193314317/Put/seqid=0 2024-12-03T02:35:15,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T02:35:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:15,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:15,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T02:35:15,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193375626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742241_1417 (size=12301) 2024-12-03T02:35:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193375634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193375646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193375646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193375646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T02:35:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:15,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:15,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193375835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193375845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193375855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193375855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193375855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:15,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T02:35:15,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:15,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:15,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:15,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T02:35:16,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/4a504e078dac445a9a0429c90db5288d 2024-12-03T02:35:16,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/05def4d51ec9499d9a317827c718ecd0 is 50, key is test_row_0/C:col10/1733193314317/Put/seqid=0 2024-12-03T02:35:16,051 INFO [Thread-1478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1478. 2024-12-03T02:35:16,051 DEBUG [Thread-1478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,051 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,051 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,052 INFO [Thread-1482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1482. 2024-12-03T02:35:16,052 DEBUG [Thread-1482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,052 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,052 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,052 INFO [Thread-1480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1480. 2024-12-03T02:35:16,053 DEBUG [Thread-1480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,053 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,053 DEBUG [Thread-1480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742242_1418 (size=12301) 2024-12-03T02:35:16,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,059 INFO [Thread-1484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1484. 2024-12-03T02:35:16,059 DEBUG [Thread-1484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,059 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,059 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,059 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-12-03T02:35:16,059 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,059 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,059 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T02:35:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:16,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59626 deadline: 1733193376144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59650 deadline: 1733193376155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59600 deadline: 1733193376165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59620 deadline: 1733193376165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59644 deadline: 1733193376165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:16,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T02:35:16,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:16,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:16,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:16,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T02:35:16,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:16,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:16,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T02:35:16,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/05def4d51ec9499d9a317827c718ecd0 2024-12-03T02:35:16,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/25fe5a3392084f1aa308a329e35c73bf as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25fe5a3392084f1aa308a329e35c73bf 2024-12-03T02:35:16,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25fe5a3392084f1aa308a329e35c73bf, entries=150, sequenceid=316, filesize=30.5 K 2024-12-03T02:35:16,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/4a504e078dac445a9a0429c90db5288d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4a504e078dac445a9a0429c90db5288d 2024-12-03T02:35:16,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4a504e078dac445a9a0429c90db5288d, entries=150, sequenceid=316, filesize=12.0 K 2024-12-03T02:35:16,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/05def4d51ec9499d9a317827c718ecd0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/05def4d51ec9499d9a317827c718ecd0 2024-12-03T02:35:16,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/05def4d51ec9499d9a317827c718ecd0, entries=150, sequenceid=316, filesize=12.0 K 2024-12-03T02:35:16,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 309d0efd241d9334329aa843e6c8f6f7 in 1009ms, sequenceid=316, compaction requested=true 2024-12-03T02:35:16,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:16,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:16,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:16,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:16,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:16,472 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:16,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 309d0efd241d9334329aa843e6c8f6f7:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:16,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:16,472 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:16,473 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:16,473 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:16,473 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/A is initiating minor compaction (all files) 2024-12-03T02:35:16,473 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/B is initiating minor compaction (all files) 2024-12-03T02:35:16,473 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/A in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,473 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/B in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,473 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/70b3295c204044d19f3ed37e49456839, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/94c27b2890544cbfb5f5aab1f5341ef9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25fe5a3392084f1aa308a329e35c73bf] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=92.3 K 2024-12-03T02:35:16,473 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/57f6d151b237413185c699ec44fd6b2e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5617b43336c746239fa9549f1bd75cce, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4a504e078dac445a9a0429c90db5288d] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=36.8 K 2024-12-03T02:35:16,473 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,473 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/70b3295c204044d19f3ed37e49456839, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/94c27b2890544cbfb5f5aab1f5341ef9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25fe5a3392084f1aa308a329e35c73bf] 2024-12-03T02:35:16,473 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 57f6d151b237413185c699ec44fd6b2e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733193311865 2024-12-03T02:35:16,473 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70b3295c204044d19f3ed37e49456839, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733193311865 2024-12-03T02:35:16,474 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 5617b43336c746239fa9549f1bd75cce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733193313097 2024-12-03T02:35:16,474 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94c27b2890544cbfb5f5aab1f5341ef9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733193313097 2024-12-03T02:35:16,474 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a504e078dac445a9a0429c90db5288d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733193314315 2024-12-03T02:35:16,474 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25fe5a3392084f1aa308a329e35c73bf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733193314315 2024-12-03T02:35:16,482 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#B#compaction#343 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:16,483 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5c3df247eb4d42fc8cba8fbba68d51be is 50, key is test_row_0/B:col10/1733193314317/Put/seqid=0 2024-12-03T02:35:16,491 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:16,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742243_1419 (size=13153) 2024-12-03T02:35:16,511 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203cc8c37e96fce4ecfa968f04c4d86f934_309d0efd241d9334329aa843e6c8f6f7 store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:16,526 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203cc8c37e96fce4ecfa968f04c4d86f934_309d0efd241d9334329aa843e6c8f6f7, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:16,526 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cc8c37e96fce4ecfa968f04c4d86f934_309d0efd241d9334329aa843e6c8f6f7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:16,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742244_1420 (size=4469) 2024-12-03T02:35:16,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T02:35:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,537 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T02:35:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:16,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:16,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cf7deb7e1bff499d9a4acd41d5320e2a_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193315492/Put/seqid=0 2024-12-03T02:35:16,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742245_1421 (size=12454) 2024-12-03T02:35:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. as already flushing 2024-12-03T02:35:16,657 INFO [Thread-1473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1473. 2024-12-03T02:35:16,657 DEBUG [Thread-1473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,657 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,657 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,666 INFO [Thread-1471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1471. 2024-12-03T02:35:16,666 DEBUG [Thread-1471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,666 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,666 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,676 INFO [Thread-1469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1469. 2024-12-03T02:35:16,676 DEBUG [Thread-1469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,676 DEBUG [Thread-1469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,676 DEBUG [Thread-1469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,676 INFO [Thread-1465 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1465. 2024-12-03T02:35:16,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,676 DEBUG [Thread-1465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,676 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,676 INFO [Thread-1467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1467. 2024-12-03T02:35:16,676 DEBUG [Thread-1467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:16,676 DEBUG [Thread-1467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,676 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,676 DEBUG [Thread-1467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:16,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:16,912 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/5c3df247eb4d42fc8cba8fbba68d51be as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5c3df247eb4d42fc8cba8fbba68d51be 2024-12-03T02:35:16,917 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/B of 309d0efd241d9334329aa843e6c8f6f7 into 5c3df247eb4d42fc8cba8fbba68d51be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:16,917 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:16,917 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/B, priority=13, startTime=1733193316472; duration=0sec 2024-12-03T02:35:16,918 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:16,918 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:B 2024-12-03T02:35:16,918 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:16,919 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:16,919 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 309d0efd241d9334329aa843e6c8f6f7/C is initiating minor compaction (all files) 2024-12-03T02:35:16,919 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 309d0efd241d9334329aa843e6c8f6f7/C in TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:16,919 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/2ac422e8d9d44027ab75c44f20d4d35b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/43ebfb21eec845e6a23230b7c2a3fad8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/05def4d51ec9499d9a317827c718ecd0] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp, totalSize=36.7 K 2024-12-03T02:35:16,919 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac422e8d9d44027ab75c44f20d4d35b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733193311865 2024-12-03T02:35:16,920 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ebfb21eec845e6a23230b7c2a3fad8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733193313097 2024-12-03T02:35:16,920 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 05def4d51ec9499d9a317827c718ecd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733193314315 2024-12-03T02:35:16,929 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#C#compaction#346 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:16,929 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/62c988c9aefc498f8b6b11644a4c8c3d is 50, key is test_row_0/C:col10/1733193314317/Put/seqid=0 2024-12-03T02:35:16,930 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 309d0efd241d9334329aa843e6c8f6f7#A#compaction#344 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:16,931 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/21787ac6d42947e5956492c61041e185 is 175, key is test_row_0/A:col10/1733193314317/Put/seqid=0 2024-12-03T02:35:16,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742246_1422 (size=13085) 2024-12-03T02:35:16,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742247_1423 (size=32107) 2024-12-03T02:35:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:16,957 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cf7deb7e1bff499d9a4acd41d5320e2a_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cf7deb7e1bff499d9a4acd41d5320e2a_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:16,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/b211241939a3463ba81c1887b59c4d55, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:16,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/b211241939a3463ba81c1887b59c4d55 is 175, key is test_row_0/A:col10/1733193315492/Put/seqid=0 2024-12-03T02:35:16,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742248_1424 (size=31255) 2024-12-03T02:35:17,343 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/21787ac6d42947e5956492c61041e185 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/21787ac6d42947e5956492c61041e185 2024-12-03T02:35:17,343 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/62c988c9aefc498f8b6b11644a4c8c3d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/62c988c9aefc498f8b6b11644a4c8c3d 2024-12-03T02:35:17,348 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/C of 309d0efd241d9334329aa843e6c8f6f7 into 62c988c9aefc498f8b6b11644a4c8c3d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:17,348 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 309d0efd241d9334329aa843e6c8f6f7/A of 309d0efd241d9334329aa843e6c8f6f7 into 21787ac6d42947e5956492c61041e185(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:17,348 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:17,348 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:17,348 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/A, priority=13, startTime=1733193316472; duration=0sec 2024-12-03T02:35:17,348 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., storeName=309d0efd241d9334329aa843e6c8f6f7/C, priority=13, startTime=1733193316472; duration=0sec 2024-12-03T02:35:17,348 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:17,348 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:A 2024-12-03T02:35:17,348 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:17,348 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 309d0efd241d9334329aa843e6c8f6f7:C 2024-12-03T02:35:17,366 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/b211241939a3463ba81c1887b59c4d55 2024-12-03T02:35:17,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/df2b9d71f9624884867413abb1c2e2fc is 50, key is test_row_0/B:col10/1733193315492/Put/seqid=0 2024-12-03T02:35:17,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742249_1425 (size=12301) 2024-12-03T02:35:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T02:35:17,779 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/df2b9d71f9624884867413abb1c2e2fc 2024-12-03T02:35:17,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/74e6e606f9774222aa94a6699acb0bf7 is 50, key is test_row_0/C:col10/1733193315492/Put/seqid=0 2024-12-03T02:35:17,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742250_1426 (size=12301) 2024-12-03T02:35:18,189 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/74e6e606f9774222aa94a6699acb0bf7 2024-12-03T02:35:18,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/b211241939a3463ba81c1887b59c4d55 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/b211241939a3463ba81c1887b59c4d55 2024-12-03T02:35:18,197 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/b211241939a3463ba81c1887b59c4d55, entries=150, sequenceid=330, filesize=30.5 K 2024-12-03T02:35:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/df2b9d71f9624884867413abb1c2e2fc as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/df2b9d71f9624884867413abb1c2e2fc 2024-12-03T02:35:18,201 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/df2b9d71f9624884867413abb1c2e2fc, entries=150, sequenceid=330, filesize=12.0 K 2024-12-03T02:35:18,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/74e6e606f9774222aa94a6699acb0bf7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/74e6e606f9774222aa94a6699acb0bf7 2024-12-03T02:35:18,205 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/74e6e606f9774222aa94a6699acb0bf7, entries=150, sequenceid=330, filesize=12.0 K 2024-12-03T02:35:18,206 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=33.54 KB/34350 for 309d0efd241d9334329aa843e6c8f6f7 in 1669ms, sequenceid=330, compaction requested=false 2024-12-03T02:35:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 309d0efd241d9334329aa843e6c8f6f7: 2024-12-03T02:35:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-03T02:35:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-03T02:35:18,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-03T02:35:18,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8960 sec 2024-12-03T02:35:18,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.9030 sec 2024-12-03T02:35:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T02:35:19,465 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1039 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3117 rows 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1047 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3137 rows 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1043 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3125 rows 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1049 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3143 rows 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1036 2024-12-03T02:35:19,465 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3105 rows 2024-12-03T02:35:19,465 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:35:19,465 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:35:19,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:19,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:19,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:19,467 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T02:35:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T02:35:19,469 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193319469"}]},"ts":"1733193319469"} 2024-12-03T02:35:19,470 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T02:35:19,470 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T02:35:19,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T02:35:19,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, UNASSIGN}] 2024-12-03T02:35:19,472 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, UNASSIGN 2024-12-03T02:35:19,473 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=309d0efd241d9334329aa843e6c8f6f7, regionState=CLOSING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:19,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, UNASSIGN because future has completed 2024-12-03T02:35:19,474 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:35:19,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:35:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T02:35:19,626 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:19,626 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:35:19,626 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 309d0efd241d9334329aa843e6c8f6f7, disabling compactions & flushes 2024-12-03T02:35:19,626 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:19,626 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:19,626 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. after waiting 0 ms 2024-12-03T02:35:19,626 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:19,626 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 309d0efd241d9334329aa843e6c8f6f7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T02:35:19,626 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=A 2024-12-03T02:35:19,627 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:19,627 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=B 2024-12-03T02:35:19,627 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:19,627 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 309d0efd241d9334329aa843e6c8f6f7, store=C 2024-12-03T02:35:19,627 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:19,632 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203eeed2ddb5eb14137b4a956a72f09dcc3_309d0efd241d9334329aa843e6c8f6f7 is 50, key is test_row_0/A:col10/1733193316675/Put/seqid=0 2024-12-03T02:35:19,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742251_1427 (size=12454) 2024-12-03T02:35:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T02:35:20,004 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T02:35:20,036 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:20,045 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203eeed2ddb5eb14137b4a956a72f09dcc3_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eeed2ddb5eb14137b4a956a72f09dcc3_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:20,048 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/aa1444ced82a47c0a16c6aed4e6f08f8, store: [table=TestAcidGuarantees family=A region=309d0efd241d9334329aa843e6c8f6f7] 2024-12-03T02:35:20,049 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/aa1444ced82a47c0a16c6aed4e6f08f8 is 175, key is test_row_0/A:col10/1733193316675/Put/seqid=0 2024-12-03T02:35:20,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742252_1428 (size=31255) 2024-12-03T02:35:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T02:35:20,454 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/aa1444ced82a47c0a16c6aed4e6f08f8 2024-12-03T02:35:20,466 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/2a0ce43584ed4c7eaa47afc8ed163774 is 50, key is test_row_0/B:col10/1733193316675/Put/seqid=0 2024-12-03T02:35:20,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742253_1429 (size=12301) 2024-12-03T02:35:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T02:35:20,877 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/2a0ce43584ed4c7eaa47afc8ed163774 2024-12-03T02:35:20,888 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/b25868e593e24348b7fd9c3c537ce73e is 50, key is test_row_0/C:col10/1733193316675/Put/seqid=0 2024-12-03T02:35:20,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742254_1430 (size=12301) 2024-12-03T02:35:21,292 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/b25868e593e24348b7fd9c3c537ce73e 2024-12-03T02:35:21,296 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/A/aa1444ced82a47c0a16c6aed4e6f08f8 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/aa1444ced82a47c0a16c6aed4e6f08f8 2024-12-03T02:35:21,299 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/aa1444ced82a47c0a16c6aed4e6f08f8, entries=150, sequenceid=341, filesize=30.5 K 2024-12-03T02:35:21,300 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/B/2a0ce43584ed4c7eaa47afc8ed163774 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/2a0ce43584ed4c7eaa47afc8ed163774 2024-12-03T02:35:21,303 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/2a0ce43584ed4c7eaa47afc8ed163774, entries=150, sequenceid=341, filesize=12.0 K 2024-12-03T02:35:21,304 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/.tmp/C/b25868e593e24348b7fd9c3c537ce73e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b25868e593e24348b7fd9c3c537ce73e 2024-12-03T02:35:21,307 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b25868e593e24348b7fd9c3c537ce73e, entries=150, sequenceid=341, filesize=12.0 K 2024-12-03T02:35:21,308 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 309d0efd241d9334329aa843e6c8f6f7 in 1682ms, sequenceid=341, compaction requested=true 2024-12-03T02:35:21,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9642949b3b024bb7980c4fabf713185a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9ae2def0a1804fecb899c4aa9cd1de23, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/d9b5faf5fb184ddbb4f9393860deb757, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/059aa1f936e9424998aac75fe2f851a7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/2744b5537b1b42c9b2a857b57124abf5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/f592dc7c718b49228606ae7ed6986f88, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/20a61c4e19e14c6fac1da7861ec6d966, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/583d2d90a45c451396dc4d8a6a90b259, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/842095f305994b4d8887999864a5fd7c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a91945940b4f42cabb0b5468afbfa69d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/75e429ce07224e508c78d3da93dec440, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/422a41a308a9467bbb4a5ae2285a17df, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9272ab09e84e4c95885e6337df689473, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a95335516b9246a988c6cf1ecd2a17d0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/ded6a2abb9fd43f790ebccc974e7849e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a742b95dc70f404d82c691311dffa675, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/035f7213ec414374952547c360ac7c1d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/bc46db65e08748d2b090e7054e478f6f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/64a04f0f72c14a96b7eeafa9c7ae1f73, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/cda861360458442ea683932b7d8769b3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/e7771c5c39264fa0925790d4b23e06d2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25403dc2901642689e5bc989046b9272, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/70b3295c204044d19f3ed37e49456839, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/94c27b2890544cbfb5f5aab1f5341ef9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25fe5a3392084f1aa308a329e35c73bf] to archive 2024-12-03T02:35:21,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:35:21,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9642949b3b024bb7980c4fabf713185a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9642949b3b024bb7980c4fabf713185a 2024-12-03T02:35:21,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9ae2def0a1804fecb899c4aa9cd1de23 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9ae2def0a1804fecb899c4aa9cd1de23 2024-12-03T02:35:21,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/d9b5faf5fb184ddbb4f9393860deb757 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/d9b5faf5fb184ddbb4f9393860deb757 2024-12-03T02:35:21,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/059aa1f936e9424998aac75fe2f851a7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/059aa1f936e9424998aac75fe2f851a7 2024-12-03T02:35:21,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/2744b5537b1b42c9b2a857b57124abf5 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/2744b5537b1b42c9b2a857b57124abf5 2024-12-03T02:35:21,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/f592dc7c718b49228606ae7ed6986f88 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/f592dc7c718b49228606ae7ed6986f88 2024-12-03T02:35:21,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/20a61c4e19e14c6fac1da7861ec6d966 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/20a61c4e19e14c6fac1da7861ec6d966 2024-12-03T02:35:21,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/583d2d90a45c451396dc4d8a6a90b259 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/583d2d90a45c451396dc4d8a6a90b259 2024-12-03T02:35:21,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/842095f305994b4d8887999864a5fd7c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/842095f305994b4d8887999864a5fd7c 2024-12-03T02:35:21,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a91945940b4f42cabb0b5468afbfa69d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a91945940b4f42cabb0b5468afbfa69d 2024-12-03T02:35:21,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/75e429ce07224e508c78d3da93dec440 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/75e429ce07224e508c78d3da93dec440 2024-12-03T02:35:21,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/422a41a308a9467bbb4a5ae2285a17df to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/422a41a308a9467bbb4a5ae2285a17df 2024-12-03T02:35:21,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9272ab09e84e4c95885e6337df689473 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/9272ab09e84e4c95885e6337df689473 2024-12-03T02:35:21,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a95335516b9246a988c6cf1ecd2a17d0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a95335516b9246a988c6cf1ecd2a17d0 2024-12-03T02:35:21,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/ded6a2abb9fd43f790ebccc974e7849e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/ded6a2abb9fd43f790ebccc974e7849e 2024-12-03T02:35:21,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a742b95dc70f404d82c691311dffa675 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/a742b95dc70f404d82c691311dffa675 2024-12-03T02:35:21,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/035f7213ec414374952547c360ac7c1d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/035f7213ec414374952547c360ac7c1d 2024-12-03T02:35:21,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/bc46db65e08748d2b090e7054e478f6f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/bc46db65e08748d2b090e7054e478f6f 2024-12-03T02:35:21,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/64a04f0f72c14a96b7eeafa9c7ae1f73 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/64a04f0f72c14a96b7eeafa9c7ae1f73 2024-12-03T02:35:21,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/cda861360458442ea683932b7d8769b3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/cda861360458442ea683932b7d8769b3 2024-12-03T02:35:21,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/e7771c5c39264fa0925790d4b23e06d2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/e7771c5c39264fa0925790d4b23e06d2 2024-12-03T02:35:21,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25403dc2901642689e5bc989046b9272 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25403dc2901642689e5bc989046b9272 2024-12-03T02:35:21,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/70b3295c204044d19f3ed37e49456839 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/70b3295c204044d19f3ed37e49456839 2024-12-03T02:35:21,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/94c27b2890544cbfb5f5aab1f5341ef9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/94c27b2890544cbfb5f5aab1f5341ef9 2024-12-03T02:35:21,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25fe5a3392084f1aa308a329e35c73bf to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/25fe5a3392084f1aa308a329e35c73bf 2024-12-03T02:35:21,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c48a0c08b1504a4da2a8727b9fe174d3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f35b753309594f32b18bff2b8b765bfa, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4f5d125f43024a4286ddcbed9d20ede7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/81aed039ccee4bd1b4196a4bdbf32075, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5a22e14b02554dd98f517c6512fc2c46, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f7e0136d227e44a9b9e46a556c5c1852, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f73cb920d0a24f1f8d351932a54f595e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/ff9d874b8cf44bcda04f0e6b6b62bb2b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5d5677613cc14ac29fd066f168bd43db, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/a51a317df2154abf8adf33a84aa625d0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/bc4ccd357e674a5d89dcd52a735593a1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/daf5b23655754919a7b2aba9ea83fd94, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/78db62060d514464972544e5396e8e2f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/13bce31cd25f4967a401e3215432cad4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/b607f1d522dc4fc3a30a7bcb89cbda46, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f78b217bf17042c590ff343f9ff426d1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c080b445bd0143ce80b835502b260b88, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/9540531a244a47768a035b8aeff2c51c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/917a30799a8b4658b625db1aa390eaa9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/05d239cc27f04d61827cd12114b77737, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/989b3c55e24e40fda19fda21de075a6a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/57f6d151b237413185c699ec44fd6b2e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/efb5b59e5fb542d0998fc98252f7b62a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5617b43336c746239fa9549f1bd75cce, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4a504e078dac445a9a0429c90db5288d] to archive 2024-12-03T02:35:21,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:35:21,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c48a0c08b1504a4da2a8727b9fe174d3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c48a0c08b1504a4da2a8727b9fe174d3 2024-12-03T02:35:21,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f35b753309594f32b18bff2b8b765bfa to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f35b753309594f32b18bff2b8b765bfa 2024-12-03T02:35:21,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4f5d125f43024a4286ddcbed9d20ede7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4f5d125f43024a4286ddcbed9d20ede7 2024-12-03T02:35:21,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/81aed039ccee4bd1b4196a4bdbf32075 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/81aed039ccee4bd1b4196a4bdbf32075 2024-12-03T02:35:21,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5a22e14b02554dd98f517c6512fc2c46 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5a22e14b02554dd98f517c6512fc2c46 2024-12-03T02:35:21,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f7e0136d227e44a9b9e46a556c5c1852 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f7e0136d227e44a9b9e46a556c5c1852 2024-12-03T02:35:21,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f73cb920d0a24f1f8d351932a54f595e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f73cb920d0a24f1f8d351932a54f595e 2024-12-03T02:35:21,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/ff9d874b8cf44bcda04f0e6b6b62bb2b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/ff9d874b8cf44bcda04f0e6b6b62bb2b 2024-12-03T02:35:21,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5d5677613cc14ac29fd066f168bd43db to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5d5677613cc14ac29fd066f168bd43db 2024-12-03T02:35:21,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/a51a317df2154abf8adf33a84aa625d0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/a51a317df2154abf8adf33a84aa625d0 2024-12-03T02:35:21,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/bc4ccd357e674a5d89dcd52a735593a1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/bc4ccd357e674a5d89dcd52a735593a1 2024-12-03T02:35:21,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/daf5b23655754919a7b2aba9ea83fd94 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/daf5b23655754919a7b2aba9ea83fd94 2024-12-03T02:35:21,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/78db62060d514464972544e5396e8e2f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/78db62060d514464972544e5396e8e2f 2024-12-03T02:35:21,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/13bce31cd25f4967a401e3215432cad4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/13bce31cd25f4967a401e3215432cad4 2024-12-03T02:35:21,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/b607f1d522dc4fc3a30a7bcb89cbda46 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/b607f1d522dc4fc3a30a7bcb89cbda46 2024-12-03T02:35:21,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f78b217bf17042c590ff343f9ff426d1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/f78b217bf17042c590ff343f9ff426d1 2024-12-03T02:35:21,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c080b445bd0143ce80b835502b260b88 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/c080b445bd0143ce80b835502b260b88 2024-12-03T02:35:21,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/9540531a244a47768a035b8aeff2c51c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/9540531a244a47768a035b8aeff2c51c 2024-12-03T02:35:21,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/917a30799a8b4658b625db1aa390eaa9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/917a30799a8b4658b625db1aa390eaa9 2024-12-03T02:35:21,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/05d239cc27f04d61827cd12114b77737 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/05d239cc27f04d61827cd12114b77737 2024-12-03T02:35:21,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/989b3c55e24e40fda19fda21de075a6a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/989b3c55e24e40fda19fda21de075a6a 2024-12-03T02:35:21,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/57f6d151b237413185c699ec44fd6b2e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/57f6d151b237413185c699ec44fd6b2e 2024-12-03T02:35:21,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/efb5b59e5fb542d0998fc98252f7b62a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/efb5b59e5fb542d0998fc98252f7b62a 2024-12-03T02:35:21,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5617b43336c746239fa9549f1bd75cce to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5617b43336c746239fa9549f1bd75cce 2024-12-03T02:35:21,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4a504e078dac445a9a0429c90db5288d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/4a504e078dac445a9a0429c90db5288d 2024-12-03T02:35:21,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b11b08ae745542aab2a5cc93faeea471, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/ca4b0969676e4b44b6accf2636ceb0cb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0f09894ba4a6415faace401973b9bcdc, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/c0e07afb22594677bd8d79e1129ecc3a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7e6cadc0f43c43e79ac2faa17f2c156f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/749cecfdfca74e8f8fbb0933bb91dcb6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8f51062029c24545852efe91fad3dc83, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/3fbd5ead21684506844d75d141c15029, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7fafe3aebfdd4dbcbb7f43afecaf54fc, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/519e2f6a4c7d474b8bdc3a1a5079ea8a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/68de51122dfc4b3585d74b8708145392, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1fa99f16295f432e860b7dbd1f45d633, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/eda3ea3d721b45bb8937a82a2da86947, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/563f0c36bdb542b19cb2252f52884e9d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8ea951353f1a446a84e94c4e4e792df6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/f1a176284a2843c5a22b57cd631eaab1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0fa222f466d5486d97ff9adc38760aea, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/d6122f2a609c472abe6dd61bd92809c7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1a9b7bc25f2c446d9c1c93f77ca98f96, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/2ac422e8d9d44027ab75c44f20d4d35b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/921e145c4181412bbefb2ad5e5389e62, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/43ebfb21eec845e6a23230b7c2a3fad8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/05def4d51ec9499d9a317827c718ecd0] to archive 2024-12-03T02:35:21,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:35:21,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b11b08ae745542aab2a5cc93faeea471 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b11b08ae745542aab2a5cc93faeea471 2024-12-03T02:35:21,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/ca4b0969676e4b44b6accf2636ceb0cb to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/ca4b0969676e4b44b6accf2636ceb0cb 2024-12-03T02:35:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0f09894ba4a6415faace401973b9bcdc to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0f09894ba4a6415faace401973b9bcdc 2024-12-03T02:35:21,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/c0e07afb22594677bd8d79e1129ecc3a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/c0e07afb22594677bd8d79e1129ecc3a 2024-12-03T02:35:21,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7e6cadc0f43c43e79ac2faa17f2c156f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7e6cadc0f43c43e79ac2faa17f2c156f 2024-12-03T02:35:21,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/749cecfdfca74e8f8fbb0933bb91dcb6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/749cecfdfca74e8f8fbb0933bb91dcb6 2024-12-03T02:35:21,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8f51062029c24545852efe91fad3dc83 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8f51062029c24545852efe91fad3dc83 2024-12-03T02:35:21,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/3fbd5ead21684506844d75d141c15029 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/3fbd5ead21684506844d75d141c15029 2024-12-03T02:35:21,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7fafe3aebfdd4dbcbb7f43afecaf54fc to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/7fafe3aebfdd4dbcbb7f43afecaf54fc 2024-12-03T02:35:21,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/519e2f6a4c7d474b8bdc3a1a5079ea8a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/519e2f6a4c7d474b8bdc3a1a5079ea8a 2024-12-03T02:35:21,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/68de51122dfc4b3585d74b8708145392 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/68de51122dfc4b3585d74b8708145392 2024-12-03T02:35:21,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1fa99f16295f432e860b7dbd1f45d633 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1fa99f16295f432e860b7dbd1f45d633 2024-12-03T02:35:21,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/eda3ea3d721b45bb8937a82a2da86947 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/eda3ea3d721b45bb8937a82a2da86947 2024-12-03T02:35:21,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/563f0c36bdb542b19cb2252f52884e9d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/563f0c36bdb542b19cb2252f52884e9d 2024-12-03T02:35:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8ea951353f1a446a84e94c4e4e792df6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/8ea951353f1a446a84e94c4e4e792df6 2024-12-03T02:35:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/f1a176284a2843c5a22b57cd631eaab1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/f1a176284a2843c5a22b57cd631eaab1 2024-12-03T02:35:21,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0fa222f466d5486d97ff9adc38760aea to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/0fa222f466d5486d97ff9adc38760aea 2024-12-03T02:35:21,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/d6122f2a609c472abe6dd61bd92809c7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/d6122f2a609c472abe6dd61bd92809c7 2024-12-03T02:35:21,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1a9b7bc25f2c446d9c1c93f77ca98f96 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/1a9b7bc25f2c446d9c1c93f77ca98f96 2024-12-03T02:35:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/2ac422e8d9d44027ab75c44f20d4d35b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/2ac422e8d9d44027ab75c44f20d4d35b 2024-12-03T02:35:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/921e145c4181412bbefb2ad5e5389e62 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/921e145c4181412bbefb2ad5e5389e62 2024-12-03T02:35:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/43ebfb21eec845e6a23230b7c2a3fad8 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/43ebfb21eec845e6a23230b7c2a3fad8 2024-12-03T02:35:21,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/05def4d51ec9499d9a317827c718ecd0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/05def4d51ec9499d9a317827c718ecd0 2024-12-03T02:35:21,378 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/recovered.edits/344.seqid, newMaxSeqId=344, maxSeqId=4 2024-12-03T02:35:21,379 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. 2024-12-03T02:35:21,379 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 309d0efd241d9334329aa843e6c8f6f7: Waiting for close lock at 1733193319626Running coprocessor pre-close hooks at 1733193319626Disabling compacts and flushes for region at 1733193319626Disabling writes for close at 1733193319626Obtaining lock to block concurrent updates at 1733193319626Preparing flush snapshotting stores in 309d0efd241d9334329aa843e6c8f6f7 at 1733193319626Finished memstore snapshotting TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733193319627 (+1 ms)Flushing stores of TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7. at 1733193319627Flushing 309d0efd241d9334329aa843e6c8f6f7/A: creating writer at 1733193319627Flushing mob file 309d0efd241d9334329aa843e6c8f6f7/A: appending metadata at 1733193319632 (+5 ms)Flushing mob file 309d0efd241d9334329aa843e6c8f6f7/A: closing flushed file at 1733193319632Flushing 309d0efd241d9334329aa843e6c8f6f7/A: appending metadata at 1733193320048 (+416 ms)Flushing 309d0efd241d9334329aa843e6c8f6f7/A: closing flushed file at 1733193320048Flushing 309d0efd241d9334329aa843e6c8f6f7/B: creating writer at 1733193320461 (+413 ms)Flushing 309d0efd241d9334329aa843e6c8f6f7/B: appending metadata at 1733193320465 (+4 ms)Flushing 309d0efd241d9334329aa843e6c8f6f7/B: closing flushed file at 1733193320465Flushing 309d0efd241d9334329aa843e6c8f6f7/C: creating writer at 1733193320884 (+419 ms)Flushing 309d0efd241d9334329aa843e6c8f6f7/C: appending metadata at 1733193320887 (+3 ms)Flushing 309d0efd241d9334329aa843e6c8f6f7/C: closing flushed file at 1733193320888 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bced4e3: reopening flushed file at 1733193321295 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22ca869e: reopening flushed file at 1733193321300 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@273e7da6: reopening flushed file at 1733193321304 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 309d0efd241d9334329aa843e6c8f6f7 in 1682ms, sequenceid=341, compaction requested=true at 1733193321308 (+4 ms)Writing region close event to WAL at 1733193321376 (+68 ms)Running coprocessor post-close hooks at 1733193321378 (+2 ms)Closed at 1733193321378 2024-12-03T02:35:21,381 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,381 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=309d0efd241d9334329aa843e6c8f6f7, regionState=CLOSED 2024-12-03T02:35:21,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:35:21,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-03T02:35:21,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 309d0efd241d9334329aa843e6c8f6f7, server=64035cbb0b46,38721,1733193202350 in 1.9090 sec 2024-12-03T02:35:21,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-03T02:35:21,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=309d0efd241d9334329aa843e6c8f6f7, UNASSIGN in 1.9140 sec 2024-12-03T02:35:21,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-03T02:35:21,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9180 sec 2024-12-03T02:35:21,391 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193321391"}]},"ts":"1733193321391"} 2024-12-03T02:35:21,393 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T02:35:21,393 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T02:35:21,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9270 sec 2024-12-03T02:35:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T02:35:21,617 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:21,618 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T02:35:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:21,621 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T02:35:21,622 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:21,624 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,627 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/recovered.edits] 2024-12-03T02:35:21,631 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/21787ac6d42947e5956492c61041e185 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/21787ac6d42947e5956492c61041e185 2024-12-03T02:35:21,633 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/aa1444ced82a47c0a16c6aed4e6f08f8 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/aa1444ced82a47c0a16c6aed4e6f08f8 2024-12-03T02:35:21,635 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/b211241939a3463ba81c1887b59c4d55 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/A/b211241939a3463ba81c1887b59c4d55 2024-12-03T02:35:21,639 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/2a0ce43584ed4c7eaa47afc8ed163774 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/2a0ce43584ed4c7eaa47afc8ed163774 2024-12-03T02:35:21,641 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5c3df247eb4d42fc8cba8fbba68d51be to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/5c3df247eb4d42fc8cba8fbba68d51be 2024-12-03T02:35:21,643 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/df2b9d71f9624884867413abb1c2e2fc to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/B/df2b9d71f9624884867413abb1c2e2fc 2024-12-03T02:35:21,645 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/62c988c9aefc498f8b6b11644a4c8c3d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/62c988c9aefc498f8b6b11644a4c8c3d 2024-12-03T02:35:21,646 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/74e6e606f9774222aa94a6699acb0bf7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/74e6e606f9774222aa94a6699acb0bf7 2024-12-03T02:35:21,647 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b25868e593e24348b7fd9c3c537ce73e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/C/b25868e593e24348b7fd9c3c537ce73e 2024-12-03T02:35:21,649 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/recovered.edits/344.seqid to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7/recovered.edits/344.seqid 2024-12-03T02:35:21,650 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,650 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T02:35:21,650 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T02:35:21,651 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T02:35:21,654 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203102ddc2f8f98463daecd6df813ffbae7_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203102ddc2f8f98463daecd6df813ffbae7_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,655 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120311cc87fd2b594cf2bd0816ec8ee38416_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120311cc87fd2b594cf2bd0816ec8ee38416_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,656 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120315e4ed8b07b14869bfe83aaff79c2a5c_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120315e4ed8b07b14869bfe83aaff79c2a5c_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,658 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f3575b58cb046eb932c4e56f2f4a7b3_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f3575b58cb046eb932c4e56f2f4a7b3_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,659 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120328fdc157d2644b6a86a12b1c5f94be55_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120328fdc157d2644b6a86a12b1c5f94be55_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,660 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034334d0f33b2f43d583562ac9071e2c0a_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034334d0f33b2f43d583562ac9071e2c0a_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,662 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120352fdb4ba1a0749fb9cdaf9ccb6b993b5_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120352fdb4ba1a0749fb9cdaf9ccb6b993b5_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,663 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035587d91a4aee459db55bfa97eb0d70d9_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035587d91a4aee459db55bfa97eb0d70d9_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,664 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036ffb26f66cb74423a710452a3f349ccb_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036ffb26f66cb74423a710452a3f349ccb_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,666 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203883f32702c5f4b66b0677c1da706c6c4_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203883f32702c5f4b66b0677c1da706c6c4_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,667 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039664b269e765436badacb35113a287d6_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039664b269e765436badacb35113a287d6_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,668 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cab3e3986114182848704e546403af9_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cab3e3986114182848704e546403af9_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,670 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b512cc5aa59e46d194865ace6e801935_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b512cc5aa59e46d194865ace6e801935_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,671 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba5abc5adaf04fb28a1a2baadc770eb5_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba5abc5adaf04fb28a1a2baadc770eb5_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,672 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c845adf92f5b41e4ba54a54eaa7c7cf5_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c845adf92f5b41e4ba54a54eaa7c7cf5_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,673 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cf7deb7e1bff499d9a4acd41d5320e2a_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cf7deb7e1bff499d9a4acd41d5320e2a_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,674 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203da0620180e0a4a719001ae0718fb1657_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203da0620180e0a4a719001ae0718fb1657_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,676 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eeed2ddb5eb14137b4a956a72f09dcc3_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eeed2ddb5eb14137b4a956a72f09dcc3_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,677 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ef076868706144cb8d3c550aa624e2a8_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ef076868706144cb8d3c550aa624e2a8_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,678 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f9099b30afdf4b11bceb25bfdceea297_309d0efd241d9334329aa843e6c8f6f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f9099b30afdf4b11bceb25bfdceea297_309d0efd241d9334329aa843e6c8f6f7 2024-12-03T02:35:21,679 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T02:35:21,681 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:21,684 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T02:35:21,686 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T02:35:21,687 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T02:35:21,688 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:21,688 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T02:35:21,688 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733193321688"}]},"ts":"9223372036854775807"} 2024-12-03T02:35:21,690 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T02:35:21,690 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 309d0efd241d9334329aa843e6c8f6f7, NAME => 'TestAcidGuarantees,,1733193292895.309d0efd241d9334329aa843e6c8f6f7.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T02:35:21,690 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T02:35:21,691 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733193321690"}]},"ts":"9223372036854775807"} 2024-12-03T02:35:21,693 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T02:35:21,693 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:21,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 75 msec 2024-12-03T02:35:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T02:35:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T02:35:21,725 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:21,732 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 248) - Thread LEAK? -, OpenFileDescriptor=459 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 812), ProcessCount=9 (was 11), AvailableMemoryMB=2732 (was 2455) - AvailableMemoryMB LEAK? - 2024-12-03T02:35:21,740 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=9, AvailableMemoryMB=2732 2024-12-03T02:35:21,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T02:35:21,742 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:35:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:21,743 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T02:35:21,744 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:21,744 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-03T02:35:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T02:35:21,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T02:35:21,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742255_1431 (size=1061) 2024-12-03T02:35:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T02:35:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T02:35:22,150 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37 2024-12-03T02:35:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742256_1432 (size=53) 2024-12-03T02:35:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T02:35:22,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:35:22,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b35fa9ee617f001aec9f096271ca1ed8, disabling compactions & flushes 2024-12-03T02:35:22,557 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:22,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:22,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. after waiting 0 ms 2024-12-03T02:35:22,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:22,557 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:22,557 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b35fa9ee617f001aec9f096271ca1ed8: Waiting for close lock at 1733193322557Disabling compacts and flushes for region at 1733193322557Disabling writes for close at 1733193322557Writing region close event to WAL at 1733193322557Closed at 1733193322557 2024-12-03T02:35:22,559 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T02:35:22,560 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733193322559"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733193322559"}]},"ts":"1733193322559"} 2024-12-03T02:35:22,563 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T02:35:22,564 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T02:35:22,564 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193322564"}]},"ts":"1733193322564"} 2024-12-03T02:35:22,566 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T02:35:22,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35fa9ee617f001aec9f096271ca1ed8, ASSIGN}] 2024-12-03T02:35:22,568 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35fa9ee617f001aec9f096271ca1ed8, ASSIGN 2024-12-03T02:35:22,569 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35fa9ee617f001aec9f096271ca1ed8, ASSIGN; state=OFFLINE, location=64035cbb0b46,38721,1733193202350; forceNewPlan=false, retain=false 2024-12-03T02:35:22,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=b35fa9ee617f001aec9f096271ca1ed8, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:22,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35fa9ee617f001aec9f096271ca1ed8, ASSIGN because future has completed 2024-12-03T02:35:22,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:35:22,878 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:22,879 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:35:22,880 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,880 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:35:22,880 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,880 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,882 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,884 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:35:22,885 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b35fa9ee617f001aec9f096271ca1ed8 columnFamilyName A 2024-12-03T02:35:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T02:35:22,885 DEBUG [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:22,886 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] regionserver.HStore(327): Store=b35fa9ee617f001aec9f096271ca1ed8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:35:22,886 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,888 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:35:22,888 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b35fa9ee617f001aec9f096271ca1ed8 columnFamilyName B 2024-12-03T02:35:22,888 DEBUG [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:22,888 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] regionserver.HStore(327): Store=b35fa9ee617f001aec9f096271ca1ed8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:35:22,888 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,890 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:35:22,890 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b35fa9ee617f001aec9f096271ca1ed8 columnFamilyName C 2024-12-03T02:35:22,890 DEBUG [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:22,890 INFO [StoreOpener-b35fa9ee617f001aec9f096271ca1ed8-1 {}] regionserver.HStore(327): Store=b35fa9ee617f001aec9f096271ca1ed8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:35:22,891 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:22,891 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,891 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,892 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,893 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,893 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,894 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:35:22,895 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,897 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:35:22,897 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened b35fa9ee617f001aec9f096271ca1ed8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67849274, jitterRate=0.011032968759536743}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:35:22,898 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:22,898 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for b35fa9ee617f001aec9f096271ca1ed8: Running coprocessor pre-open hook at 1733193322880Writing region info on filesystem at 1733193322880Initializing all the Stores at 1733193322882 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193322882Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193322882Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193322882Cleaning up temporary data from old regions at 1733193322893 (+11 ms)Running coprocessor post-open hooks at 1733193322898 (+5 ms)Region opened successfully at 1733193322898 2024-12-03T02:35:22,899 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., pid=114, masterSystemTime=1733193322874 2024-12-03T02:35:22,901 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:22,901 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:22,901 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=b35fa9ee617f001aec9f096271ca1ed8, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:22,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:35:22,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-03T02:35:22,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 in 181 msec 2024-12-03T02:35:22,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-03T02:35:22,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35fa9ee617f001aec9f096271ca1ed8, ASSIGN in 338 msec 2024-12-03T02:35:22,906 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T02:35:22,906 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193322906"}]},"ts":"1733193322906"} 2024-12-03T02:35:22,907 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T02:35:22,908 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T02:35:22,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-12-03T02:35:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T02:35:23,895 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:23,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd2060e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,898 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224020fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,899 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,901 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T02:35:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 64035cbb0b46,44739,1733193201066 2024-12-03T02:35:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@225ef102 2024-12-03T02:35:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T02:35:23,902 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T02:35:23,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,904 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422b2d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5408d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,906 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680b1d5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,907 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b00982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d98347, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@437af99c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc07c53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600aadee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a8a2978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0274bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,916 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7272cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,917 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5707bdfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,919 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48487887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f783c13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,923 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca73de3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,924 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55272bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,925 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c52249d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,926 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6dea72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:23,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:23,928 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d98a9a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:23,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:23,929 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e35da14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:23,930 DEBUG [Thread-1944 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,930 DEBUG [Thread-1959 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,930 DEBUG [Thread-1953 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,930 DEBUG [Thread-1948 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,931 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:23,931 DEBUG [Thread-1942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,931 DEBUG [Thread-1955 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,931 DEBUG [Thread-1946 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-03T02:35:23,932 DEBUG [Thread-1961 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,932 DEBUG [Thread-1955 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,932 DEBUG [Thread-1955 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,933 DEBUG [Thread-1950 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,933 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T02:35:23,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,935 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:23,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,938 DEBUG [Thread-1957 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:23,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:23,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2] 2024-12-03T02:35:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:23,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T02:35:23,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:23,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:23,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:23,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:23,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:23,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:23,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/26bbc85863b8488c8154c53aa042ed0b is 50, key is test_row_0/A:col10/1733193323944/Put/seqid=0 2024-12-03T02:35:23,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193383974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193383975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:23,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193323971, max timestamp=1733193323980Num uniques 859; 2024-12-03T02:35:23,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:23,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193323971, max timestamp=1733193323980Num uniques 859; 2024-12-03T02:35:23,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193323971, max timestamp=1733193323980Num uniques 859; 2024-12-03T02:35:23,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:23,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:23,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742257_1433 (size=14341) 2024-12-03T02:35:23,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/26bbc85863b8488c8154c53aa042ed0b 2024-12-03T02:35:24,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/4e1ff90e78cb41d8b1e661bd3dc95b60 is 50, key is test_row_0/B:col10/1733193323944/Put/seqid=0 2024-12-03T02:35:24,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742258_1434 (size=12001) 2024-12-03T02:35:24,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/4e1ff90e78cb41d8b1e661bd3dc95b60 2024-12-03T02:35:24,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193384027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/11011d1a3dc048339c916872ad515b3e is 50, key is test_row_0/C:col10/1733193323944/Put/seqid=0 2024-12-03T02:35:24,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742259_1435 (size=12001) 2024-12-03T02:35:24,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/11011d1a3dc048339c916872ad515b3e 2024-12-03T02:35:24,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193384032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193384031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T02:35:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/26bbc85863b8488c8154c53aa042ed0b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/26bbc85863b8488c8154c53aa042ed0b 2024-12-03T02:35:24,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/26bbc85863b8488c8154c53aa042ed0b, entries=200, sequenceid=12, filesize=14.0 K 2024-12-03T02:35:24,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/4e1ff90e78cb41d8b1e661bd3dc95b60 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/4e1ff90e78cb41d8b1e661bd3dc95b60 2024-12-03T02:35:24,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/4e1ff90e78cb41d8b1e661bd3dc95b60, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T02:35:24,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/11011d1a3dc048339c916872ad515b3e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/11011d1a3dc048339c916872ad515b3e 2024-12-03T02:35:24,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/11011d1a3dc048339c916872ad515b3e, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T02:35:24,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b35fa9ee617f001aec9f096271ca1ed8 in 125ms, sequenceid=12, compaction requested=false 2024-12-03T02:35:24,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:24,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-03T02:35:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:24,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T02:35:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:24,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:24,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/85fd94f2bc114711a6349c1f3262ca81 is 50, key is test_row_0/A:col10/1733193324031/Put/seqid=0 2024-12-03T02:35:24,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742260_1436 (size=14341) 2024-12-03T02:35:24,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/85fd94f2bc114711a6349c1f3262ca81 2024-12-03T02:35:24,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/9506ca6a3f8a432796be85f03875b53d is 50, key is test_row_0/B:col10/1733193324031/Put/seqid=0 2024-12-03T02:35:24,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742261_1437 (size=12001) 2024-12-03T02:35:24,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/9506ca6a3f8a432796be85f03875b53d 2024-12-03T02:35:24,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/66e321f148834ee0b5b745d07505875c is 50, key is test_row_0/C:col10/1733193324031/Put/seqid=0 2024-12-03T02:35:24,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742262_1438 (size=12001) 2024-12-03T02:35:24,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193384119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193384122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193384145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193384154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193384154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193384225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193384235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T02:35:24,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:24,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:24,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:24,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:24,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T02:35:24,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193384365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193384365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193384365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T02:35:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:24,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:24,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:24,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193384435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193384444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/66e321f148834ee0b5b745d07505875c 2024-12-03T02:35:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/85fd94f2bc114711a6349c1f3262ca81 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/85fd94f2bc114711a6349c1f3262ca81 2024-12-03T02:35:24,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/85fd94f2bc114711a6349c1f3262ca81, entries=200, sequenceid=47, filesize=14.0 K 2024-12-03T02:35:24,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/9506ca6a3f8a432796be85f03875b53d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9506ca6a3f8a432796be85f03875b53d 2024-12-03T02:35:24,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9506ca6a3f8a432796be85f03875b53d, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T02:35:24,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/66e321f148834ee0b5b745d07505875c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/66e321f148834ee0b5b745d07505875c 2024-12-03T02:35:24,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/66e321f148834ee0b5b745d07505875c, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T02:35:24,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for b35fa9ee617f001aec9f096271ca1ed8 in 453ms, sequenceid=47, compaction requested=false 2024-12-03T02:35:24,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:24,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T02:35:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:24,550 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:35:24,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:24,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:24,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/5afb394ca0b240818bcb94d0b282df25 is 50, key is test_row_0/A:col10/1733193324111/Put/seqid=0 2024-12-03T02:35:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T02:35:24,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742263_1439 (size=9657) 2024-12-03T02:35:24,567 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/5afb394ca0b240818bcb94d0b282df25 2024-12-03T02:35:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/fbea8cf8d4c44999bf8bbfc833a2258d is 50, key is test_row_0/B:col10/1733193324111/Put/seqid=0 2024-12-03T02:35:24,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742264_1440 (size=9657) 2024-12-03T02:35:24,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:24,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193324697, max timestamp=1733193324717Num uniques 859; 2024-12-03T02:35:24,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:35:24,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193324697, max timestamp=1733193324717Num uniques 859; 2024-12-03T02:35:24,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:35:24,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193324697, max timestamp=1733193324717Num uniques 859; 2024-12-03T02:35:24,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:35:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193384764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733193324753, max timestamp=1733193324767Num uniques 1018; 2024-12-03T02:35:24,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733193324753, max timestamp=1733193324767Num uniques 1018; 2024-12-03T02:35:24,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T02:35:24,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T02:35:24,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733193324753, max timestamp=1733193324767Num uniques 1018; 2024-12-03T02:35:24,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T02:35:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193384798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193384799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193384804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193384805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,904 WARN [regionserver/64035cbb0b46:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-03T02:35:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193384914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193384914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193384915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193384925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:24,985 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/fbea8cf8d4c44999bf8bbfc833a2258d 2024-12-03T02:35:24,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/632fb5640a714cb98ccdd0d749195c92 is 50, key is test_row_0/C:col10/1733193324111/Put/seqid=0 2024-12-03T02:35:24,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742265_1441 (size=9657) 2024-12-03T02:35:24,995 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/632fb5640a714cb98ccdd0d749195c92 2024-12-03T02:35:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/5afb394ca0b240818bcb94d0b282df25 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5afb394ca0b240818bcb94d0b282df25 2024-12-03T02:35:25,001 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5afb394ca0b240818bcb94d0b282df25, entries=100, sequenceid=57, filesize=9.4 K 2024-12-03T02:35:25,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/fbea8cf8d4c44999bf8bbfc833a2258d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fbea8cf8d4c44999bf8bbfc833a2258d 2024-12-03T02:35:25,007 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fbea8cf8d4c44999bf8bbfc833a2258d, entries=100, sequenceid=57, filesize=9.4 K 2024-12-03T02:35:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/632fb5640a714cb98ccdd0d749195c92 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/632fb5640a714cb98ccdd0d749195c92 2024-12-03T02:35:25,011 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/632fb5640a714cb98ccdd0d749195c92, entries=100, sequenceid=57, filesize=9.4 K 2024-12-03T02:35:25,012 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b35fa9ee617f001aec9f096271ca1ed8 in 462ms, sequenceid=57, compaction requested=true 2024-12-03T02:35:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-03T02:35:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-03T02:35:25,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-03T02:35:25,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0780 sec 2024-12-03T02:35:25,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0840 sec 2024-12-03T02:35:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T02:35:25,074 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:25,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-03T02:35:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T02:35:25,076 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:25,077 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:25,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:25,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:25,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=140.89 KB heapSize=394.69 KB 2024-12-03T02:35:25,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:25,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:25,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:25,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:25,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:25,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:25,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/bb2a22968d0d46b5b47ebb4c9e3d028a is 50, key is test_row_0/A:col10/1733193324804/Put/seqid=0 2024-12-03T02:35:25,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742266_1442 (size=14341) 2024-12-03T02:35:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193385142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193385142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193385143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193385144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T02:35:25,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T02:35:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193385254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193385254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193385264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193385264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193385275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T02:35:25,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:25,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T02:35:25,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193385464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193385464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193385475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193385475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T02:35:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/bb2a22968d0d46b5b47ebb4c9e3d028a 2024-12-03T02:35:25,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/02b4daf7f0ed4e73a1e210e14002ad38 is 50, key is test_row_0/B:col10/1733193324804/Put/seqid=0 2024-12-03T02:35:25,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742267_1443 (size=12001) 2024-12-03T02:35:25,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/02b4daf7f0ed4e73a1e210e14002ad38 2024-12-03T02:35:25,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/b2a32b5ecd4846cfac4c02b1b412aca5 is 50, key is test_row_0/C:col10/1733193324804/Put/seqid=0 2024-12-03T02:35:25,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742268_1444 (size=12001) 2024-12-03T02:35:25,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T02:35:25,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T02:35:25,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193385774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193385774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193385785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193385785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:25,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T02:35:25,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:25,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:25,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/b2a32b5ecd4846cfac4c02b1b412aca5 2024-12-03T02:35:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/bb2a22968d0d46b5b47ebb4c9e3d028a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/bb2a22968d0d46b5b47ebb4c9e3d028a 2024-12-03T02:35:25,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/bb2a22968d0d46b5b47ebb4c9e3d028a, entries=200, sequenceid=110, filesize=14.0 K 2024-12-03T02:35:25,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/02b4daf7f0ed4e73a1e210e14002ad38 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/02b4daf7f0ed4e73a1e210e14002ad38 2024-12-03T02:35:25,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/02b4daf7f0ed4e73a1e210e14002ad38, entries=150, sequenceid=110, filesize=11.7 K 2024-12-03T02:35:25,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/b2a32b5ecd4846cfac4c02b1b412aca5 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b2a32b5ecd4846cfac4c02b1b412aca5 2024-12-03T02:35:25,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b2a32b5ecd4846cfac4c02b1b412aca5, entries=150, sequenceid=110, filesize=11.7 K 2024-12-03T02:35:25,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~394.64 KB/404112, currentSize=40.25 KB/41220 for b35fa9ee617f001aec9f096271ca1ed8 in 842ms, sequenceid=110, compaction requested=true 2024-12-03T02:35:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:25,974 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:25,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:25,974 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:25,975 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:25,975 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:25,975 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:25,975 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:25,975 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,975 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,975 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/26bbc85863b8488c8154c53aa042ed0b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/85fd94f2bc114711a6349c1f3262ca81, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5afb394ca0b240818bcb94d0b282df25, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/bb2a22968d0d46b5b47ebb4c9e3d028a] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=51.4 K 2024-12-03T02:35:25,975 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/4e1ff90e78cb41d8b1e661bd3dc95b60, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9506ca6a3f8a432796be85f03875b53d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fbea8cf8d4c44999bf8bbfc833a2258d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/02b4daf7f0ed4e73a1e210e14002ad38] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=44.6 K 2024-12-03T02:35:25,975 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1ff90e78cb41d8b1e661bd3dc95b60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733193323944 2024-12-03T02:35:25,975 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26bbc85863b8488c8154c53aa042ed0b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733193323943 2024-12-03T02:35:25,975 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9506ca6a3f8a432796be85f03875b53d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733193324028 2024-12-03T02:35:25,976 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85fd94f2bc114711a6349c1f3262ca81, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733193324014 2024-12-03T02:35:25,976 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting fbea8cf8d4c44999bf8bbfc833a2258d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733193324111 2024-12-03T02:35:25,976 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5afb394ca0b240818bcb94d0b282df25, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733193324111 2024-12-03T02:35:25,976 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb2a22968d0d46b5b47ebb4c9e3d028a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733193324796 2024-12-03T02:35:25,976 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b4daf7f0ed4e73a1e210e14002ad38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733193324798 2024-12-03T02:35:25,984 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:25,984 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/c507ae6169bf4f29906571cf783c1123 is 50, key is test_row_0/A:col10/1733193324804/Put/seqid=0 2024-12-03T02:35:25,985 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:25,986 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/8f7ab715731749788b2b8a4161a19b10 is 50, key is test_row_0/B:col10/1733193324804/Put/seqid=0 2024-12-03T02:35:25,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742269_1445 (size=12139) 2024-12-03T02:35:25,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742270_1446 (size=12139) 2024-12-03T02:35:25,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T02:35:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:25,992 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T02:35:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:25,994 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/8f7ab715731749788b2b8a4161a19b10 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8f7ab715731749788b2b8a4161a19b10 2024-12-03T02:35:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/4574aceb5f534eb0941d3396f633172e is 50, key is test_row_0/A:col10/1733193325141/Put/seqid=0 2024-12-03T02:35:25,998 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into 8f7ab715731749788b2b8a4161a19b10(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:25,998 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:25,998 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=12, startTime=1733193325974; duration=0sec 2024-12-03T02:35:25,999 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:25,999 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:25,999 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:26,002 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:26,002 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:26,002 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:26,002 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/11011d1a3dc048339c916872ad515b3e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/66e321f148834ee0b5b745d07505875c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/632fb5640a714cb98ccdd0d749195c92, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b2a32b5ecd4846cfac4c02b1b412aca5] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=44.6 K 2024-12-03T02:35:26,002 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 11011d1a3dc048339c916872ad515b3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733193323944 2024-12-03T02:35:26,003 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 66e321f148834ee0b5b745d07505875c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733193324028 2024-12-03T02:35:26,003 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 632fb5640a714cb98ccdd0d749195c92, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733193324111 2024-12-03T02:35:26,003 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b2a32b5ecd4846cfac4c02b1b412aca5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733193324798 2024-12-03T02:35:26,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742271_1447 (size=12001) 2024-12-03T02:35:26,010 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:26,010 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/7a69a056ae2942939c53e18ecce92d0d is 50, key is test_row_0/C:col10/1733193324804/Put/seqid=0 2024-12-03T02:35:26,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742272_1448 (size=12139) 2024-12-03T02:35:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T02:35:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:26,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:26,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733193326306, max timestamp=1733193326328Num uniques 859; 2024-12-03T02:35:26,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733193326306, max timestamp=1733193326328Num uniques 859; 2024-12-03T02:35:26,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T02:35:26,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733193326306, max timestamp=1733193326328Num uniques 859; 2024-12-03T02:35:26,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T02:35:26,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T02:35:26,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193386353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193386353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193386354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193386354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193386355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,397 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/c507ae6169bf4f29906571cf783c1123 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/c507ae6169bf4f29906571cf783c1123 2024-12-03T02:35:26,401 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into c507ae6169bf4f29906571cf783c1123(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:26,401 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:26,401 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=12, startTime=1733193325974; duration=0sec 2024-12-03T02:35:26,401 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:26,401 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:26,405 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/4574aceb5f534eb0941d3396f633172e 2024-12-03T02:35:26,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/3bf1441aa00b4374a3f08f4f3b60bf78 is 50, key is test_row_0/B:col10/1733193325141/Put/seqid=0 2024-12-03T02:35:26,418 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/7a69a056ae2942939c53e18ecce92d0d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7a69a056ae2942939c53e18ecce92d0d 2024-12-03T02:35:26,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742273_1449 (size=12001) 2024-12-03T02:35:26,421 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/3bf1441aa00b4374a3f08f4f3b60bf78 2024-12-03T02:35:26,423 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into 7a69a056ae2942939c53e18ecce92d0d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:26,423 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:26,423 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=12, startTime=1733193325974; duration=0sec 2024-12-03T02:35:26,423 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:26,423 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:26,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/b3eeac80600b49d3b7aec41ccf633c8a is 50, key is test_row_0/C:col10/1733193325141/Put/seqid=0 2024-12-03T02:35:26,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742274_1450 (size=12001) 2024-12-03T02:35:26,441 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/b3eeac80600b49d3b7aec41ccf633c8a 2024-12-03T02:35:26,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/4574aceb5f534eb0941d3396f633172e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/4574aceb5f534eb0941d3396f633172e 2024-12-03T02:35:26,450 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/4574aceb5f534eb0941d3396f633172e, entries=150, sequenceid=119, filesize=11.7 K 2024-12-03T02:35:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/3bf1441aa00b4374a3f08f4f3b60bf78 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/3bf1441aa00b4374a3f08f4f3b60bf78 2024-12-03T02:35:26,455 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/3bf1441aa00b4374a3f08f4f3b60bf78, entries=150, sequenceid=119, filesize=11.7 K 2024-12-03T02:35:26,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/b3eeac80600b49d3b7aec41ccf633c8a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b3eeac80600b49d3b7aec41ccf633c8a 2024-12-03T02:35:26,459 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b3eeac80600b49d3b7aec41ccf633c8a, entries=150, sequenceid=119, filesize=11.7 K 2024-12-03T02:35:26,459 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for b35fa9ee617f001aec9f096271ca1ed8 in 467ms, sequenceid=119, compaction requested=false 2024-12-03T02:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-03T02:35:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-03T02:35:26,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-03T02:35:26,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3840 sec 2024-12-03T02:35:26,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.3870 sec 2024-12-03T02:35:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:26,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-12-03T02:35:26,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:26,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:26,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:26,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:26,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:26,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:26,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/e84cd211b5df43769dc8f7e2b2ab4a7a is 50, key is test_row_0/A:col10/1733193326479/Put/seqid=0 2024-12-03T02:35:26,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742275_1451 (size=14541) 2024-12-03T02:35:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193386483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193386484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193386486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193386488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193386494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193386605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193386605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193386605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193386605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193386605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193386824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193386824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193386824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193386824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193386824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:26,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/e84cd211b5df43769dc8f7e2b2ab4a7a 2024-12-03T02:35:26,899 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:35:26,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/9cb79802939d451e978dc37bdb618a1c is 50, key is test_row_0/B:col10/1733193326479/Put/seqid=0 2024-12-03T02:35:26,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742276_1452 (size=12151) 2024-12-03T02:35:27,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193387145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193387145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193387145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193387145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193387146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T02:35:27,235 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:27,235 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-03T02:35:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T02:35:27,237 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:27,238 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:27,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:27,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/9cb79802939d451e978dc37bdb618a1c 2024-12-03T02:35:27,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/8cdee0a2f6354a56a1347d91a7e9a45a is 50, key is test_row_0/C:col10/1733193326479/Put/seqid=0 2024-12-03T02:35:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T02:35:27,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742277_1453 (size=12151) 2024-12-03T02:35:27,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/8cdee0a2f6354a56a1347d91a7e9a45a 2024-12-03T02:35:27,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/e84cd211b5df43769dc8f7e2b2ab4a7a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e84cd211b5df43769dc8f7e2b2ab4a7a 2024-12-03T02:35:27,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e84cd211b5df43769dc8f7e2b2ab4a7a, entries=200, sequenceid=158, filesize=14.2 K 2024-12-03T02:35:27,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/9cb79802939d451e978dc37bdb618a1c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9cb79802939d451e978dc37bdb618a1c 2024-12-03T02:35:27,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9cb79802939d451e978dc37bdb618a1c, entries=150, sequenceid=158, filesize=11.9 K 2024-12-03T02:35:27,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/8cdee0a2f6354a56a1347d91a7e9a45a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8cdee0a2f6354a56a1347d91a7e9a45a 2024-12-03T02:35:27,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8cdee0a2f6354a56a1347d91a7e9a45a, entries=150, sequenceid=158, filesize=11.9 K 2024-12-03T02:35:27,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for b35fa9ee617f001aec9f096271ca1ed8 in 901ms, sequenceid=158, compaction requested=true 2024-12-03T02:35:27,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:27,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:27,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:27,382 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:27,382 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:27,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:27,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:27,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:27,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:27,384 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38681 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:27,384 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:27,385 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:27,385 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/c507ae6169bf4f29906571cf783c1123, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/4574aceb5f534eb0941d3396f633172e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e84cd211b5df43769dc8f7e2b2ab4a7a] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=37.8 K 2024-12-03T02:35:27,384 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:27,385 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:27,385 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:27,385 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8f7ab715731749788b2b8a4161a19b10, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/3bf1441aa00b4374a3f08f4f3b60bf78, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9cb79802939d451e978dc37bdb618a1c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=35.4 K 2024-12-03T02:35:27,385 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting c507ae6169bf4f29906571cf783c1123, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733193324798 2024-12-03T02:35:27,385 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f7ab715731749788b2b8a4161a19b10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733193324798 2024-12-03T02:35:27,386 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4574aceb5f534eb0941d3396f633172e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733193325141 2024-12-03T02:35:27,386 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e84cd211b5df43769dc8f7e2b2ab4a7a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733193326353 2024-12-03T02:35:27,388 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bf1441aa00b4374a3f08f4f3b60bf78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733193325141 2024-12-03T02:35:27,388 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb79802939d451e978dc37bdb618a1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733193326353 2024-12-03T02:35:27,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T02:35:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:27,397 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T02:35:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:27,414 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:27,415 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/70ecaf20c4db431ebc427bcfa4f5ad88 is 50, key is test_row_0/A:col10/1733193326479/Put/seqid=0 2024-12-03T02:35:27,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/f60af39a933341a4acbe5a2899e606c7 is 50, key is test_row_0/A:col10/1733193326482/Put/seqid=0 2024-12-03T02:35:27,420 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:27,421 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/b794e1cf0b1149f3b7828e88d0699518 is 50, key is test_row_0/B:col10/1733193326479/Put/seqid=0 2024-12-03T02:35:27,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742278_1454 (size=12151) 2024-12-03T02:35:27,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742279_1455 (size=12391) 2024-12-03T02:35:27,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742280_1456 (size=12391) 2024-12-03T02:35:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T02:35:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:27,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:27,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193327726, max timestamp=1733193327758Num uniques 859; 2024-12-03T02:35:27,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:27,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193327726, max timestamp=1733193327758Num uniques 859; 2024-12-03T02:35:27,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:27,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193327726, max timestamp=1733193327758Num uniques 859; 2024-12-03T02:35:27,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:27,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193387791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193387792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193387793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193387794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193387801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,839 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/f60af39a933341a4acbe5a2899e606c7 2024-12-03T02:35:27,841 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/70ecaf20c4db431ebc427bcfa4f5ad88 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/70ecaf20c4db431ebc427bcfa4f5ad88 2024-12-03T02:35:27,848 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into 70ecaf20c4db431ebc427bcfa4f5ad88(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:27,848 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:27,848 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=13, startTime=1733193327382; duration=0sec 2024-12-03T02:35:27,848 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:27,848 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:27,848 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:27,849 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:27,849 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:27,849 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:27,849 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7a69a056ae2942939c53e18ecce92d0d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b3eeac80600b49d3b7aec41ccf633c8a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8cdee0a2f6354a56a1347d91a7e9a45a] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=35.4 K 2024-12-03T02:35:27,850 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a69a056ae2942939c53e18ecce92d0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733193324798 2024-12-03T02:35:27,850 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3eeac80600b49d3b7aec41ccf633c8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733193325141 2024-12-03T02:35:27,850 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cdee0a2f6354a56a1347d91a7e9a45a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733193326353 2024-12-03T02:35:27,858 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/b794e1cf0b1149f3b7828e88d0699518 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b794e1cf0b1149f3b7828e88d0699518 2024-12-03T02:35:27,862 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into b794e1cf0b1149f3b7828e88d0699518(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:27,862 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:27,862 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=13, startTime=1733193327382; duration=0sec 2024-12-03T02:35:27,863 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:27,863 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/8dbb7662a85d45468f29be3128c650bb is 50, key is test_row_0/B:col10/1733193326482/Put/seqid=0 2024-12-03T02:35:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T02:35:27,867 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:27,867 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/4c2a2a4e2b034723b73b4a73412a95df is 50, key is test_row_0/C:col10/1733193326479/Put/seqid=0 2024-12-03T02:35:27,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742281_1457 (size=12151) 2024-12-03T02:35:27,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742282_1458 (size=12391) 2024-12-03T02:35:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193387915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193387915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193387915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193387915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193387915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193388134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193388135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193388135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193388135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193388135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,273 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/8dbb7662a85d45468f29be3128c650bb 2024-12-03T02:35:28,290 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/4c2a2a4e2b034723b73b4a73412a95df as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c2a2a4e2b034723b73b4a73412a95df 2024-12-03T02:35:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/fd7db597c0e44382b3420e682fc8d217 is 50, key is test_row_0/C:col10/1733193326482/Put/seqid=0 2024-12-03T02:35:28,302 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into 4c2a2a4e2b034723b73b4a73412a95df(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:28,302 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:28,302 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=13, startTime=1733193327383; duration=0sec 2024-12-03T02:35:28,302 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:28,302 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:28,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742283_1459 (size=12151) 2024-12-03T02:35:28,308 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/fd7db597c0e44382b3420e682fc8d217 2024-12-03T02:35:28,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/f60af39a933341a4acbe5a2899e606c7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f60af39a933341a4acbe5a2899e606c7 2024-12-03T02:35:28,320 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f60af39a933341a4acbe5a2899e606c7, entries=150, sequenceid=166, filesize=11.9 K 2024-12-03T02:35:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/8dbb7662a85d45468f29be3128c650bb as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8dbb7662a85d45468f29be3128c650bb 2024-12-03T02:35:28,328 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8dbb7662a85d45468f29be3128c650bb, entries=150, sequenceid=166, filesize=11.9 K 2024-12-03T02:35:28,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/fd7db597c0e44382b3420e682fc8d217 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/fd7db597c0e44382b3420e682fc8d217 2024-12-03T02:35:28,336 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/fd7db597c0e44382b3420e682fc8d217, entries=150, sequenceid=166, filesize=11.9 K 2024-12-03T02:35:28,337 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for b35fa9ee617f001aec9f096271ca1ed8 in 939ms, sequenceid=166, compaction requested=false 2024-12-03T02:35:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-03T02:35:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-03T02:35:28,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-03T02:35:28,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1000 sec 2024-12-03T02:35:28,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.1070 sec 2024-12-03T02:35:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T02:35:28,375 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:28,375 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-03T02:35:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:35:28,377 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:28,377 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:28,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:28,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=154.31 KB heapSize=436.08 KB 2024-12-03T02:35:28,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:28,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:35:28,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:28,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:35:28,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:28,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:35:28,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/63a5e1633665445cb49d2c783c322269 is 50, key is test_row_0/A:col10/1733193327789/Put/seqid=0 2024-12-03T02:35:28,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742284_1460 (size=14541) 2024-12-03T02:35:28,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193388463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193388466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193388466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193388467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193388467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:35:28,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T02:35:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:28,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193388585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193388585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193388585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193388585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193388585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T02:35:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:28,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:35:28,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193388808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193388809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193388809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193388809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193388809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:28,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T02:35:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:28,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/63a5e1633665445cb49d2c783c322269 2024-12-03T02:35:28,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/9fb554d6aec3466cb6ebc6aae8aba428 is 50, key is test_row_0/B:col10/1733193327789/Put/seqid=0 2024-12-03T02:35:28,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742285_1461 (size=12151) 2024-12-03T02:35:28,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T02:35:28,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:28,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:28,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:28,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:35:29,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193389125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193389125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193389125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193389125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193389125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T02:35:29,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:29,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T02:35:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/9fb554d6aec3466cb6ebc6aae8aba428 2024-12-03T02:35:29,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/73a68c2fcbf34383903e4dafd67a35ce is 50, key is test_row_0/C:col10/1733193327789/Put/seqid=0 2024-12-03T02:35:29,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742286_1462 (size=12151) 2024-12-03T02:35:29,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T02:35:29,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:35:29,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T02:35:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:29,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193389635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193389635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193389635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193389635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193389635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:29,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/73a68c2fcbf34383903e4dafd67a35ce 2024-12-03T02:35:29,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/63a5e1633665445cb49d2c783c322269 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/63a5e1633665445cb49d2c783c322269 2024-12-03T02:35:29,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/63a5e1633665445cb49d2c783c322269, entries=200, sequenceid=207, filesize=14.2 K 2024-12-03T02:35:29,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/9fb554d6aec3466cb6ebc6aae8aba428 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9fb554d6aec3466cb6ebc6aae8aba428 2024-12-03T02:35:29,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9fb554d6aec3466cb6ebc6aae8aba428, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T02:35:29,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/73a68c2fcbf34383903e4dafd67a35ce as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/73a68c2fcbf34383903e4dafd67a35ce 2024-12-03T02:35:29,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/73a68c2fcbf34383903e4dafd67a35ce, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T02:35:29,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=26.84 KB/27480 for b35fa9ee617f001aec9f096271ca1ed8 in 1285ms, sequenceid=207, compaction requested=true 2024-12-03T02:35:29,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:29,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:29,748 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:29,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:29,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:29,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:29,748 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:29,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:29,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:29,749 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:29,749 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:29,749 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:29,749 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:29,749 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,749 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,749 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b794e1cf0b1149f3b7828e88d0699518, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8dbb7662a85d45468f29be3128c650bb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9fb554d6aec3466cb6ebc6aae8aba428] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=35.8 K 2024-12-03T02:35:29,749 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/70ecaf20c4db431ebc427bcfa4f5ad88, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f60af39a933341a4acbe5a2899e606c7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/63a5e1633665445cb49d2c783c322269] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=38.2 K 2024-12-03T02:35:29,749 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b794e1cf0b1149f3b7828e88d0699518, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733193326353 2024-12-03T02:35:29,749 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70ecaf20c4db431ebc427bcfa4f5ad88, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733193326353 2024-12-03T02:35:29,749 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbb7662a85d45468f29be3128c650bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733193326482 2024-12-03T02:35:29,749 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting f60af39a933341a4acbe5a2899e606c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733193326482 2024-12-03T02:35:29,750 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a5e1633665445cb49d2c783c322269, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733193327789 2024-12-03T02:35:29,750 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb554d6aec3466cb6ebc6aae8aba428, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733193327789 2024-12-03T02:35:29,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T02:35:29,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,752 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T02:35:29,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:29,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:29,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:29,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:29,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/b059257bb9d94736ae511eb98fefb67e is 50, key is test_row_1/A:col10/1733193328465/Put/seqid=0 2024-12-03T02:35:29,773 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:29,774 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:29,775 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/f3ce05a5fa6f4ab2aeb95bfc69c7c5ff is 50, key is test_row_0/A:col10/1733193327789/Put/seqid=0 2024-12-03T02:35:29,775 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/af6608b0a2ce4ecbaea07b8dda758395 is 50, key is test_row_0/B:col10/1733193327789/Put/seqid=0 2024-12-03T02:35:29,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742289_1465 (size=12493) 2024-12-03T02:35:29,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742287_1463 (size=9757) 2024-12-03T02:35:29,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742288_1464 (size=12493) 2024-12-03T02:35:29,830 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/f3ce05a5fa6f4ab2aeb95bfc69c7c5ff as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f3ce05a5fa6f4ab2aeb95bfc69c7c5ff 2024-12-03T02:35:29,835 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into f3ce05a5fa6f4ab2aeb95bfc69c7c5ff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:29,835 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:29,835 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=13, startTime=1733193329748; duration=0sec 2024-12-03T02:35:29,835 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:29,835 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:29,835 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:29,836 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:29,836 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:29,836 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:29,836 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c2a2a4e2b034723b73b4a73412a95df, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/fd7db597c0e44382b3420e682fc8d217, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/73a68c2fcbf34383903e4dafd67a35ce] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=35.8 K 2024-12-03T02:35:29,836 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c2a2a4e2b034723b73b4a73412a95df, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733193326353 2024-12-03T02:35:29,837 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd7db597c0e44382b3420e682fc8d217, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733193326482 2024-12-03T02:35:29,837 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73a68c2fcbf34383903e4dafd67a35ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733193327789 2024-12-03T02:35:29,843 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:29,844 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/4379ecbe47124eeeb698bf17831a4200 is 50, key is test_row_0/C:col10/1733193327789/Put/seqid=0 2024-12-03T02:35:29,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742290_1466 (size=12493) 2024-12-03T02:35:30,187 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/af6608b0a2ce4ecbaea07b8dda758395 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/af6608b0a2ce4ecbaea07b8dda758395 2024-12-03T02:35:30,190 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into af6608b0a2ce4ecbaea07b8dda758395(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:30,190 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:30,190 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=13, startTime=1733193329748; duration=0sec 2024-12-03T02:35:30,190 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:30,190 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:30,209 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/b059257bb9d94736ae511eb98fefb67e 2024-12-03T02:35:30,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/d5d919c67b7442a4981513e95ba46108 is 50, key is test_row_1/B:col10/1733193328465/Put/seqid=0 2024-12-03T02:35:30,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742291_1467 (size=9757) 2024-12-03T02:35:30,254 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/4379ecbe47124eeeb698bf17831a4200 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4379ecbe47124eeeb698bf17831a4200 2024-12-03T02:35:30,258 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into 4379ecbe47124eeeb698bf17831a4200(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:30,258 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:30,258 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=13, startTime=1733193329748; duration=0sec 2024-12-03T02:35:30,258 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:30,258 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:35:30,626 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/d5d919c67b7442a4981513e95ba46108 2024-12-03T02:35:30,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/5973b977d58046b393f838adf2c41e77 is 50, key is test_row_1/C:col10/1733193328465/Put/seqid=0 2024-12-03T02:35:30,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742292_1468 (size=9757) 2024-12-03T02:35:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:30,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:30,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733193330665, max timestamp=1733193330682Num uniques 859; 2024-12-03T02:35:30,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T02:35:30,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733193330665, max timestamp=1733193330682Num uniques 859; 2024-12-03T02:35:30,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T02:35:30,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733193330665, max timestamp=1733193330682Num uniques 859; 2024-12-03T02:35:30,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T02:35:30,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193390710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:30,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733193330701, max timestamp=1733193330717Num uniques 1168; 2024-12-03T02:35:30,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733193330701, max timestamp=1733193330717Num uniques 1168; 2024-12-03T02:35:30,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-12-03T02:35:30,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-12-03T02:35:30,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733193330701, max timestamp=1733193330717Num uniques 1168; 2024-12-03T02:35:30,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-12-03T02:35:30,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193390752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:30,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733193330749, max timestamp=1733193330762Num uniques 1177; 2024-12-03T02:35:30,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-12-03T02:35:30,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733193330749, max timestamp=1733193330762Num uniques 1177; 2024-12-03T02:35:30,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-12-03T02:35:30,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733193330750, max timestamp=1733193330762Num uniques 1177; 2024-12-03T02:35:30,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=8102, suffix heap size=150376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=177 2024-12-03T02:35:30,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733193330773, max timestamp=1733193330846Num uniques 1086; 2024-12-03T02:35:30,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-12-03T02:35:30,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1733193330773, max timestamp=1733193330846Num uniques 1086; 2024-12-03T02:35:30,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-12-03T02:35:30,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733193330828, max timestamp=1733193330846Num uniques 1036; 2024-12-03T02:35:30,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-03T02:35:30,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8910, totalHeapSize=20520, min timestamp=1733193330904, max timestamp=1733193330935Num uniques 1195; 2024-12-03T02:35:30,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54710, new segment data size=8910, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-12-03T02:35:30,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1733193330904, max timestamp=1733193330935Num uniques 1045; 2024-12-03T02:35:30,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=7964, suffix heap size=136648,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=174 2024-12-03T02:35:30,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8093, totalHeapSize=18648, min timestamp=1733193330904, max timestamp=1733193330935Num uniques 1195; 2024-12-03T02:35:30,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54711, new segment data size=8093, suffix heap size=152248,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=177 2024-12-03T02:35:30,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733193330982, max timestamp=1733193330996Num uniques 1033; 2024-12-03T02:35:30,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733193330982, max timestamp=1733193330996Num uniques 1036; 2024-12-03T02:35:30,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47314, new segment data size=13920, suffix heap size=135400,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=304 2024-12-03T02:35:30,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47452, new segment data size=13920, suffix heap size=135712,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=304 2024-12-03T02:35:31,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733193330982, max timestamp=1733193330996Num uniques 1054; 2024-12-03T02:35:31,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-12-03T02:35:31,038 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/5973b977d58046b393f838adf2c41e77 2024-12-03T02:35:31,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/b059257bb9d94736ae511eb98fefb67e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b059257bb9d94736ae511eb98fefb67e 2024-12-03T02:35:31,045 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b059257bb9d94736ae511eb98fefb67e, entries=100, sequenceid=214, filesize=9.5 K 2024-12-03T02:35:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/d5d919c67b7442a4981513e95ba46108 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/d5d919c67b7442a4981513e95ba46108 2024-12-03T02:35:31,048 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/d5d919c67b7442a4981513e95ba46108, entries=100, sequenceid=214, filesize=9.5 K 2024-12-03T02:35:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/5973b977d58046b393f838adf2c41e77 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/5973b977d58046b393f838adf2c41e77 2024-12-03T02:35:31,052 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/5973b977d58046b393f838adf2c41e77, entries=100, sequenceid=214, filesize=9.5 K 2024-12-03T02:35:31,061 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=154.31 KB/158010 for b35fa9ee617f001aec9f096271ca1ed8 in 1309ms, sequenceid=214, compaction requested=false 2024-12-03T02:35:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:31,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-03T02:35:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:31,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=174.43 KB heapSize=491.63 KB 2024-12-03T02:35:31,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:31,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:35:31,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:31,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:35:31,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-03T02:35:31,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:35:31,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-03T02:35:31,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6860 sec 2024-12-03T02:35:31,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6900 sec 2024-12-03T02:35:31,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/e45ad668f3674da2b5182bd86c521e17 is 50, key is test_row_0/A:col10/1733193331049/Put/seqid=0 2024-12-03T02:35:31,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742293_1469 (size=12301) 2024-12-03T02:35:31,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193391062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193391063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193391063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193391064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193391064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193391174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193391175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193391175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193391185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193391185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193391385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193391395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193391395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193391404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193391405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/e45ad668f3674da2b5182bd86c521e17 2024-12-03T02:35:31,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/2da7fd869b2b4c8cb66f1cc0998d2e74 is 50, key is test_row_0/B:col10/1733193331049/Put/seqid=0 2024-12-03T02:35:31,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742294_1470 (size=12301) 2024-12-03T02:35:31,687 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T02:35:31,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T02:35:31,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193391695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193391705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193391714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193391714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193391725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:31,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/2da7fd869b2b4c8cb66f1cc0998d2e74 2024-12-03T02:35:31,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/7dbed7ae1d8642eb842f104cbe670126 is 50, key is test_row_0/C:col10/1733193331049/Put/seqid=0 2024-12-03T02:35:31,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742295_1471 (size=12301) 2024-12-03T02:35:32,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193392205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193392215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193392226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193392235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:32,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193392235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:32,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/7dbed7ae1d8642eb842f104cbe670126 2024-12-03T02:35:32,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/e45ad668f3674da2b5182bd86c521e17 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e45ad668f3674da2b5182bd86c521e17 2024-12-03T02:35:32,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e45ad668f3674da2b5182bd86c521e17, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T02:35:32,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/2da7fd869b2b4c8cb66f1cc0998d2e74 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/2da7fd869b2b4c8cb66f1cc0998d2e74 2024-12-03T02:35:32,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/2da7fd869b2b4c8cb66f1cc0998d2e74, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T02:35:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/7dbed7ae1d8642eb842f104cbe670126 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7dbed7ae1d8642eb842f104cbe670126 2024-12-03T02:35:32,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7dbed7ae1d8642eb842f104cbe670126, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T02:35:32,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~491.58 KB/503376, currentSize=13.42 KB/13740 for b35fa9ee617f001aec9f096271ca1ed8 in 1357ms, sequenceid=343, compaction requested=true 2024-12-03T02:35:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:32,418 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:32,418 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:32,419 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:32,419 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:32,419 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:32,419 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/af6608b0a2ce4ecbaea07b8dda758395, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/d5d919c67b7442a4981513e95ba46108, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/2da7fd869b2b4c8cb66f1cc0998d2e74] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=33.7 K 2024-12-03T02:35:32,419 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:32,419 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:32,419 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:32,420 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f3ce05a5fa6f4ab2aeb95bfc69c7c5ff, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b059257bb9d94736ae511eb98fefb67e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e45ad668f3674da2b5182bd86c521e17] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=33.7 K 2024-12-03T02:35:32,420 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting af6608b0a2ce4ecbaea07b8dda758395, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733193327789 2024-12-03T02:35:32,420 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ce05a5fa6f4ab2aeb95bfc69c7c5ff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733193327789 2024-12-03T02:35:32,420 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting b059257bb9d94736ae511eb98fefb67e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733193328465 2024-12-03T02:35:32,420 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d5d919c67b7442a4981513e95ba46108, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733193328465 2024-12-03T02:35:32,421 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e45ad668f3674da2b5182bd86c521e17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193331029 2024-12-03T02:35:32,421 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da7fd869b2b4c8cb66f1cc0998d2e74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193331029 2024-12-03T02:35:32,428 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:32,428 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:32,428 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/6866dd8e8de2423f90f39f97c5366253 is 50, key is test_row_0/A:col10/1733193331049/Put/seqid=0 2024-12-03T02:35:32,428 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/bff6806072a7477f805c0062900941f1 is 50, key is test_row_0/B:col10/1733193331049/Put/seqid=0 2024-12-03T02:35:32,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742296_1472 (size=12745) 2024-12-03T02:35:32,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742297_1473 (size=12745) 2024-12-03T02:35:32,472 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/6866dd8e8de2423f90f39f97c5366253 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/6866dd8e8de2423f90f39f97c5366253 2024-12-03T02:35:32,478 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/bff6806072a7477f805c0062900941f1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/bff6806072a7477f805c0062900941f1 2024-12-03T02:35:32,490 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into bff6806072a7477f805c0062900941f1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:32,490 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:32,490 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=13, startTime=1733193332418; duration=0sec 2024-12-03T02:35:32,490 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:32,490 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:32,490 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:32,491 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:32,491 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:32,491 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:32,491 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4379ecbe47124eeeb698bf17831a4200, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/5973b977d58046b393f838adf2c41e77, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7dbed7ae1d8642eb842f104cbe670126] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=33.7 K 2024-12-03T02:35:32,491 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 4379ecbe47124eeeb698bf17831a4200, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733193327789 2024-12-03T02:35:32,492 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 5973b977d58046b393f838adf2c41e77, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733193328465 2024-12-03T02:35:32,492 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dbed7ae1d8642eb842f104cbe670126, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193331029 2024-12-03T02:35:32,492 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into 6866dd8e8de2423f90f39f97c5366253(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:32,492 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:32,492 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=13, startTime=1733193332418; duration=0sec 2024-12-03T02:35:32,493 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:32,493 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:32,499 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:32,500 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/54a4da55950a4b14853a01f201fb95b8 is 50, key is test_row_0/C:col10/1733193331049/Put/seqid=0 2024-12-03T02:35:32,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742298_1474 (size=12745) 2024-12-03T02:35:32,532 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/54a4da55950a4b14853a01f201fb95b8 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/54a4da55950a4b14853a01f201fb95b8 2024-12-03T02:35:32,538 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into 54a4da55950a4b14853a01f201fb95b8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:32,539 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:32,539 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=13, startTime=1733193332418; duration=0sec 2024-12-03T02:35:32,539 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:32,539 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:35:32,551 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:32,556 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-03T02:35:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T02:35:32,561 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:32,561 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:32,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T02:35:32,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T02:35:32,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:32,721 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-03T02:35:32,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:32,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/69ca27cc0e9d4d99a972ef54c4ee4b98 is 50, key is test_row_0/A:col10/1733193331062/Put/seqid=0 2024-12-03T02:35:32,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742299_1475 (size=7415) 2024-12-03T02:35:32,736 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/69ca27cc0e9d4d99a972ef54c4ee4b98 2024-12-03T02:35:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/162e4446ffd540e5bd27f38debd48eba is 50, key is test_row_0/B:col10/1733193331062/Put/seqid=0 2024-12-03T02:35:32,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742300_1476 (size=7415) 2024-12-03T02:35:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T02:35:33,179 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/162e4446ffd540e5bd27f38debd48eba 2024-12-03T02:35:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T02:35:33,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/2a57811a31a64aba95289adb124f5633 is 50, key is test_row_0/C:col10/1733193331062/Put/seqid=0 2024-12-03T02:35:33,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:33,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742301_1477 (size=7415) 2024-12-03T02:35:33,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193333273, max timestamp=1733193333290Num uniques 859; 2024-12-03T02:35:33,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:33,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193333273, max timestamp=1733193333290Num uniques 859; 2024-12-03T02:35:33,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193333273, max timestamp=1733193333290Num uniques 859; 2024-12-03T02:35:33,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:33,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:33,295 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/2a57811a31a64aba95289adb124f5633 2024-12-03T02:35:33,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/69ca27cc0e9d4d99a972ef54c4ee4b98 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/69ca27cc0e9d4d99a972ef54c4ee4b98 2024-12-03T02:35:33,311 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/69ca27cc0e9d4d99a972ef54c4ee4b98, entries=50, sequenceid=351, filesize=7.2 K 2024-12-03T02:35:33,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/162e4446ffd540e5bd27f38debd48eba as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/162e4446ffd540e5bd27f38debd48eba 2024-12-03T02:35:33,320 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/162e4446ffd540e5bd27f38debd48eba, entries=50, sequenceid=351, filesize=7.2 K 2024-12-03T02:35:33,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/2a57811a31a64aba95289adb124f5633 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/2a57811a31a64aba95289adb124f5633 2024-12-03T02:35:33,328 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/2a57811a31a64aba95289adb124f5633, entries=50, sequenceid=351, filesize=7.2 K 2024-12-03T02:35:33,329 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=107.34 KB/109920 for b35fa9ee617f001aec9f096271ca1ed8 in 607ms, sequenceid=351, compaction requested=false 2024-12-03T02:35:33,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:33,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:33,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-03T02:35:33,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-03T02:35:33,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-03T02:35:33,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 769 msec 2024-12-03T02:35:33,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 777 msec 2024-12-03T02:35:33,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=114.05 KB heapSize=315.84 KB 2024-12-03T02:35:33,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:33,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:33,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:33,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:33,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:33,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:33,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/19ec8550e9684213ad7cc45d918c2691 is 50, key is test_row_0/A:col10/1733193333309/Put/seqid=0 2024-12-03T02:35:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193393375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193393378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193393379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193393380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742302_1478 (size=14741) 2024-12-03T02:35:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193393396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193393506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193393506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193393507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193393507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193393531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T02:35:33,695 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:33,695 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-03T02:35:33,697 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T02:35:33,698 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:33,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:33,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193393714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193393714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193393714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193393725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193393754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:33,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T02:35:33,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/19ec8550e9684213ad7cc45d918c2691 2024-12-03T02:35:33,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/fa5bf3a930d2434d9f269a6782cce850 is 50, key is test_row_0/B:col10/1733193333309/Put/seqid=0 2024-12-03T02:35:33,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742303_1479 (size=12301) 2024-12-03T02:35:33,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T02:35:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:33,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:33,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:34,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T02:35:34,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:34,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:34,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:34,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:34,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T02:35:34,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193394025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193394026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193394035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193394035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193394065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T02:35:34,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:34,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:34,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:34,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:34,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:34,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/fa5bf3a930d2434d9f269a6782cce850 2024-12-03T02:35:34,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/9f269b072ab447478a25f647e769407a is 50, key is test_row_0/C:col10/1733193333309/Put/seqid=0 2024-12-03T02:35:34,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742304_1480 (size=12301) 2024-12-03T02:35:34,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/9f269b072ab447478a25f647e769407a 2024-12-03T02:35:34,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/19ec8550e9684213ad7cc45d918c2691 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/19ec8550e9684213ad7cc45d918c2691 2024-12-03T02:35:34,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/19ec8550e9684213ad7cc45d918c2691, entries=200, sequenceid=383, filesize=14.4 K 2024-12-03T02:35:34,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/fa5bf3a930d2434d9f269a6782cce850 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fa5bf3a930d2434d9f269a6782cce850 2024-12-03T02:35:34,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fa5bf3a930d2434d9f269a6782cce850, entries=150, sequenceid=383, filesize=12.0 K 2024-12-03T02:35:34,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/9f269b072ab447478a25f647e769407a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f269b072ab447478a25f647e769407a 2024-12-03T02:35:34,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f269b072ab447478a25f647e769407a, entries=150, sequenceid=383, filesize=12.0 K 2024-12-03T02:35:34,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~315.80 KB/323376, currentSize=67.09 KB/68700 for b35fa9ee617f001aec9f096271ca1ed8 in 956ms, sequenceid=383, compaction requested=true 2024-12-03T02:35:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:34,307 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:34,307 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:34,308 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34901 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:34,308 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32461 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:34,308 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:34,308 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:34,308 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:34,308 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:34,308 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/6866dd8e8de2423f90f39f97c5366253, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/69ca27cc0e9d4d99a972ef54c4ee4b98, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/19ec8550e9684213ad7cc45d918c2691] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=34.1 K 2024-12-03T02:35:34,308 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/bff6806072a7477f805c0062900941f1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/162e4446ffd540e5bd27f38debd48eba, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fa5bf3a930d2434d9f269a6782cce850] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=31.7 K 2024-12-03T02:35:34,308 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6866dd8e8de2423f90f39f97c5366253, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193331029 2024-12-03T02:35:34,308 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting bff6806072a7477f805c0062900941f1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193331029 2024-12-03T02:35:34,309 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 162e4446ffd540e5bd27f38debd48eba, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733193331062 2024-12-03T02:35:34,309 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69ca27cc0e9d4d99a972ef54c4ee4b98, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733193331062 2024-12-03T02:35:34,311 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting fa5bf3a930d2434d9f269a6782cce850, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733193333309 2024-12-03T02:35:34,311 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19ec8550e9684213ad7cc45d918c2691, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733193333307 2024-12-03T02:35:34,317 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:34,317 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:34,317 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/0bc3c685a4d44e379bf23caae26fc082 is 50, key is test_row_0/A:col10/1733193333309/Put/seqid=0 2024-12-03T02:35:34,317 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/b33fe2b47680404dbb32afbf4a63066b is 50, key is test_row_0/B:col10/1733193333309/Put/seqid=0 2024-12-03T02:35:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T02:35:34,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742306_1482 (size=12847) 2024-12-03T02:35:34,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T02:35:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:34,334 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T02:35:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:34,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742305_1481 (size=12847) 2024-12-03T02:35:34,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/cd80d3c8a5414e07a7c8fed489b8d232 is 50, key is test_row_0/A:col10/1733193333378/Put/seqid=0 2024-12-03T02:35:34,348 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/b33fe2b47680404dbb32afbf4a63066b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b33fe2b47680404dbb32afbf4a63066b 2024-12-03T02:35:34,353 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into b33fe2b47680404dbb32afbf4a63066b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:34,353 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:34,353 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=13, startTime=1733193334307; duration=0sec 2024-12-03T02:35:34,353 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:34,353 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:34,353 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:34,356 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32461 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:34,356 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:34,356 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:34,356 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/54a4da55950a4b14853a01f201fb95b8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/2a57811a31a64aba95289adb124f5633, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f269b072ab447478a25f647e769407a] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=31.7 K 2024-12-03T02:35:34,357 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 54a4da55950a4b14853a01f201fb95b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193331029 2024-12-03T02:35:34,357 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a57811a31a64aba95289adb124f5633, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733193331062 2024-12-03T02:35:34,358 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f269b072ab447478a25f647e769407a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733193333309 2024-12-03T02:35:34,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742307_1483 (size=12301) 2024-12-03T02:35:34,393 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:34,394 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/ce6477c7840b43b7a46621298d26e671 is 50, key is test_row_0/C:col10/1733193333309/Put/seqid=0 2024-12-03T02:35:34,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742308_1484 (size=12847) 2024-12-03T02:35:34,420 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/ce6477c7840b43b7a46621298d26e671 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/ce6477c7840b43b7a46621298d26e671 2024-12-03T02:35:34,428 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into ce6477c7840b43b7a46621298d26e671(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:34,428 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:34,428 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=13, startTime=1733193334307; duration=0sec 2024-12-03T02:35:34,428 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:34,428 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:34,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:34,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193394566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193394570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193394571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193394574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193394575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193394685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193394685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193394685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193394685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,744 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/0bc3c685a4d44e379bf23caae26fc082 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0bc3c685a4d44e379bf23caae26fc082 2024-12-03T02:35:34,747 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into 0bc3c685a4d44e379bf23caae26fc082(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:34,747 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:34,747 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=13, startTime=1733193334307; duration=0sec 2024-12-03T02:35:34,747 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:34,747 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:34,770 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/cd80d3c8a5414e07a7c8fed489b8d232 2024-12-03T02:35:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/67a2f2fc9ae84ad5907af988ebbfd7e6 is 50, key is test_row_0/B:col10/1733193333378/Put/seqid=0 2024-12-03T02:35:34,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742309_1485 (size=12301) 2024-12-03T02:35:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T02:35:34,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193394895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193394895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193394895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:34,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193394904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,180 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/67a2f2fc9ae84ad5907af988ebbfd7e6 2024-12-03T02:35:35,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193395205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193395206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193395207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/b71c6114771b444dad9ebc0c7b4fb73c is 50, key is test_row_0/C:col10/1733193333378/Put/seqid=0 2024-12-03T02:35:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193395225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742310_1486 (size=12301) 2024-12-03T02:35:35,267 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/b71c6114771b444dad9ebc0c7b4fb73c 2024-12-03T02:35:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/cd80d3c8a5414e07a7c8fed489b8d232 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/cd80d3c8a5414e07a7c8fed489b8d232 2024-12-03T02:35:35,334 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/cd80d3c8a5414e07a7c8fed489b8d232, entries=150, sequenceid=396, filesize=12.0 K 2024-12-03T02:35:35,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/67a2f2fc9ae84ad5907af988ebbfd7e6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/67a2f2fc9ae84ad5907af988ebbfd7e6 2024-12-03T02:35:35,396 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/67a2f2fc9ae84ad5907af988ebbfd7e6, entries=150, sequenceid=396, filesize=12.0 K 2024-12-03T02:35:35,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/b71c6114771b444dad9ebc0c7b4fb73c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b71c6114771b444dad9ebc0c7b4fb73c 2024-12-03T02:35:35,413 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b71c6114771b444dad9ebc0c7b4fb73c, entries=150, sequenceid=396, filesize=12.0 K 2024-12-03T02:35:35,415 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b35fa9ee617f001aec9f096271ca1ed8 in 1081ms, sequenceid=396, compaction requested=false 2024-12-03T02:35:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-03T02:35:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-03T02:35:35,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-03T02:35:35,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7230 sec 2024-12-03T02:35:35,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7340 sec 2024-12-03T02:35:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:35,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T02:35:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:35,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:35,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/b08a46ab5c464533910d91c7e09ab33f is 50, key is test_row_0/A:col10/1733193335596/Put/seqid=0 2024-12-03T02:35:35,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742311_1487 (size=14741) 2024-12-03T02:35:35,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/b08a46ab5c464533910d91c7e09ab33f 2024-12-03T02:35:35,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/7d3d2d55357b4c6da5936d5a48e80200 is 50, key is test_row_0/B:col10/1733193335596/Put/seqid=0 2024-12-03T02:35:35,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193395699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193395714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193395715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742312_1488 (size=12301) 2024-12-03T02:35:35,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/7d3d2d55357b4c6da5936d5a48e80200 2024-12-03T02:35:35,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193395725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193395744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/446c25a5cad4482487b2ce279090c486 is 50, key is test_row_0/C:col10/1733193335596/Put/seqid=0 2024-12-03T02:35:35,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193395836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T02:35:35,855 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:35,860 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-03T02:35:35,866 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T02:35:35,867 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:35,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:35,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742313_1489 (size=12301) 2024-12-03T02:35:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T02:35:36,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T02:35:36,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:36,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:36,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:36,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:36,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:36,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193396055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:36,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T02:35:36,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:36,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:36,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:36,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:36,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T02:35:36,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/446c25a5cad4482487b2ce279090c486 2024-12-03T02:35:36,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/b08a46ab5c464533910d91c7e09ab33f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b08a46ab5c464533910d91c7e09ab33f 2024-12-03T02:35:36,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b08a46ab5c464533910d91c7e09ab33f, entries=200, sequenceid=419, filesize=14.4 K 2024-12-03T02:35:36,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/7d3d2d55357b4c6da5936d5a48e80200 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d3d2d55357b4c6da5936d5a48e80200 2024-12-03T02:35:36,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d3d2d55357b4c6da5936d5a48e80200, entries=150, sequenceid=419, filesize=12.0 K 2024-12-03T02:35:36,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/446c25a5cad4482487b2ce279090c486 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/446c25a5cad4482487b2ce279090c486 2024-12-03T02:35:36,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/446c25a5cad4482487b2ce279090c486, entries=150, sequenceid=419, filesize=12.0 K 2024-12-03T02:35:36,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b35fa9ee617f001aec9f096271ca1ed8 in 715ms, sequenceid=419, compaction requested=true 2024-12-03T02:35:36,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:36,313 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:36,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:36,314 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:36,314 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:36,314 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:36,314 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0bc3c685a4d44e379bf23caae26fc082, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/cd80d3c8a5414e07a7c8fed489b8d232, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b08a46ab5c464533910d91c7e09ab33f] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=39.0 K 2024-12-03T02:35:36,315 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc3c685a4d44e379bf23caae26fc082, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733193333309 2024-12-03T02:35:36,316 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd80d3c8a5414e07a7c8fed489b8d232, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733193333374 2024-12-03T02:35:36,316 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting b08a46ab5c464533910d91c7e09ab33f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733193334569 2024-12-03T02:35:36,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:36,316 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:36,317 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:36,317 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:36,317 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:36,317 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b33fe2b47680404dbb32afbf4a63066b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/67a2f2fc9ae84ad5907af988ebbfd7e6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d3d2d55357b4c6da5936d5a48e80200] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=36.6 K 2024-12-03T02:35:36,318 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b33fe2b47680404dbb32afbf4a63066b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733193333309 2024-12-03T02:35:36,318 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 67a2f2fc9ae84ad5907af988ebbfd7e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733193333374 2024-12-03T02:35:36,319 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d3d2d55357b4c6da5936d5a48e80200, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733193334569 2024-12-03T02:35:36,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:36,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:36,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:36,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:36,330 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:36,330 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/28244b971e5b47ed9729d21d320b9e59 is 50, key is test_row_0/A:col10/1733193335596/Put/seqid=0 2024-12-03T02:35:36,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T02:35:36,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:36,332 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T02:35:36,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:36,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:36,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:36,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:36,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:36,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:36,334 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:36,335 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/6b9d374ab6604a4db28296d94d22c1c0 is 50, key is test_row_0/B:col10/1733193335596/Put/seqid=0 2024-12-03T02:35:36,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/dd625a5a82c44df68c289cc850f767fb is 50, key is test_row_0/A:col10/1733193335687/Put/seqid=0 2024-12-03T02:35:36,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742315_1491 (size=12949) 2024-12-03T02:35:36,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742314_1490 (size=12949) 2024-12-03T02:35:36,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742316_1492 (size=12301) 2024-12-03T02:35:36,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:36,389 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/dd625a5a82c44df68c289cc850f767fb 2024-12-03T02:35:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:36,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/742d698bf1d24855a4b2531e631838d4 is 50, key is test_row_0/B:col10/1733193335687/Put/seqid=0 2024-12-03T02:35:36,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742317_1493 (size=12301) 2024-12-03T02:35:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T02:35:36,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193396585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:36,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193396704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:36,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193396727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:36,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193396727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:36,738 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in b35fa9ee617f001aec9f096271ca1ed8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:36,739 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in b35fa9ee617f001aec9f096271ca1ed8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193396756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:36,759 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in b35fa9ee617f001aec9f096271ca1ed8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:36,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193396765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:36,774 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in b35fa9ee617f001aec9f096271ca1ed8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:36,810 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/28244b971e5b47ed9729d21d320b9e59 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/28244b971e5b47ed9729d21d320b9e59 2024-12-03T02:35:36,811 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/6b9d374ab6604a4db28296d94d22c1c0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/6b9d374ab6604a4db28296d94d22c1c0 2024-12-03T02:35:36,819 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into 28244b971e5b47ed9729d21d320b9e59(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:36,819 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:36,819 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=13, startTime=1733193336313; duration=0sec 2024-12-03T02:35:36,819 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:36,819 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:36,820 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:36,820 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into 6b9d374ab6604a4db28296d94d22c1c0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:36,820 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:36,820 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=13, startTime=1733193336316; duration=0sec 2024-12-03T02:35:36,820 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:36,820 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:36,821 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:36,822 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:36,822 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:36,822 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/ce6477c7840b43b7a46621298d26e671, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b71c6114771b444dad9ebc0c7b4fb73c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/446c25a5cad4482487b2ce279090c486] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=36.6 K 2024-12-03T02:35:36,822 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6477c7840b43b7a46621298d26e671, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733193333309 2024-12-03T02:35:36,825 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting b71c6114771b444dad9ebc0c7b4fb73c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733193333374 2024-12-03T02:35:36,827 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 446c25a5cad4482487b2ce279090c486, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733193334569 2024-12-03T02:35:36,841 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/742d698bf1d24855a4b2531e631838d4 2024-12-03T02:35:36,845 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:36,846 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/9f4908225d234803a85b9bf510c27144 is 50, key is test_row_0/C:col10/1733193335596/Put/seqid=0 2024-12-03T02:35:36,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/1f3fdc0af95f431e847f2fb45b49be57 is 50, key is test_row_0/C:col10/1733193335687/Put/seqid=0 2024-12-03T02:35:36,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742318_1494 (size=12949) 2024-12-03T02:35:36,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742319_1495 (size=12301) 2024-12-03T02:35:36,916 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/1f3fdc0af95f431e847f2fb45b49be57 2024-12-03T02:35:36,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193396915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:36,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/dd625a5a82c44df68c289cc850f767fb as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dd625a5a82c44df68c289cc850f767fb 2024-12-03T02:35:36,972 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dd625a5a82c44df68c289cc850f767fb, entries=150, sequenceid=431, filesize=12.0 K 2024-12-03T02:35:36,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/742d698bf1d24855a4b2531e631838d4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/742d698bf1d24855a4b2531e631838d4 2024-12-03T02:35:36,979 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/742d698bf1d24855a4b2531e631838d4, entries=150, sequenceid=431, filesize=12.0 K 2024-12-03T02:35:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/1f3fdc0af95f431e847f2fb45b49be57 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1f3fdc0af95f431e847f2fb45b49be57 2024-12-03T02:35:36,985 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1f3fdc0af95f431e847f2fb45b49be57, entries=150, sequenceid=431, filesize=12.0 K 2024-12-03T02:35:36,988 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b35fa9ee617f001aec9f096271ca1ed8 in 656ms, sequenceid=431, compaction requested=false 2024-12-03T02:35:36,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:36,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:36,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-03T02:35:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-03T02:35:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T02:35:37,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-03T02:35:37,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1360 sec 2024-12-03T02:35:37,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.1530 sec 2024-12-03T02:35:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:37,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193336521, max timestamp=1733193337241Num uniques 859; 2024-12-03T02:35:37,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:35:37,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193336521, max timestamp=1733193337241Num uniques 859; 2024-12-03T02:35:37,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=89.06 KB heapSize=261.20 KB 2024-12-03T02:35:37,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:35:37,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:37,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:37,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:37,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:37,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:37,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:37,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/e7cbbd74a90d4b41ac54e1db41197a88 is 50, key is test_row_0/A:col10/1733193336561/Put/seqid=0 2024-12-03T02:35:37,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742320_1496 (size=12301) 2024-12-03T02:35:37,317 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/9f4908225d234803a85b9bf510c27144 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f4908225d234803a85b9bf510c27144 2024-12-03T02:35:37,344 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into 9f4908225d234803a85b9bf510c27144(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:37,344 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:37,344 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=13, startTime=1733193336322; duration=0sec 2024-12-03T02:35:37,344 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:37,344 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:37,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193337410, max timestamp=1733193337449Num uniques 859; 2024-12-03T02:35:37,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193337410, max timestamp=1733193337449Num uniques 859; 2024-12-03T02:35:37,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:35:37,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:35:37,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193337410, max timestamp=1733193337449Num uniques 859; 2024-12-03T02:35:37,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:35:37,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193397605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:37,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/e7cbbd74a90d4b41ac54e1db41197a88 2024-12-03T02:35:37,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193397731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:37,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/c62267068d9e433c8dfb434c042a743c is 50, key is test_row_0/B:col10/1733193336561/Put/seqid=0 2024-12-03T02:35:37,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742321_1497 (size=12301) 2024-12-03T02:35:37,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/c62267068d9e433c8dfb434c042a743c 2024-12-03T02:35:37,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/a6f76c32d15f43b8b0d44b447f0ceaa7 is 50, key is test_row_0/C:col10/1733193336561/Put/seqid=0 2024-12-03T02:35:37,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742322_1498 (size=12301) 2024-12-03T02:35:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193397954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T02:35:38,024 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:38,025 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-03T02:35:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T02:35:38,027 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:38,028 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:38,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T02:35:38,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T02:35:38,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:38,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:38,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:38,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:38,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:38,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/a6f76c32d15f43b8b0d44b447f0ceaa7 2024-12-03T02:35:38,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/e7cbbd74a90d4b41ac54e1db41197a88 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e7cbbd74a90d4b41ac54e1db41197a88 2024-12-03T02:35:38,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e7cbbd74a90d4b41ac54e1db41197a88, entries=150, sequenceid=454, filesize=12.0 K 2024-12-03T02:35:38,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/c62267068d9e433c8dfb434c042a743c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c62267068d9e433c8dfb434c042a743c 2024-12-03T02:35:38,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193398275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:38,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c62267068d9e433c8dfb434c042a743c, entries=150, sequenceid=454, filesize=12.0 K 2024-12-03T02:35:38,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/a6f76c32d15f43b8b0d44b447f0ceaa7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/a6f76c32d15f43b8b0d44b447f0ceaa7 2024-12-03T02:35:38,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/a6f76c32d15f43b8b0d44b447f0ceaa7, entries=150, sequenceid=454, filesize=12.0 K 2024-12-03T02:35:38,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~57.35 KB/58730, heapSize ~162.30 KB/166192, currentSize=127.47 KB/130530 for b35fa9ee617f001aec9f096271ca1ed8 in 1072ms, sequenceid=454, compaction requested=true 2024-12-03T02:35:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:38,323 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:38,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:35:38,323 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:38,327 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:38,327 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:38,327 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:38,328 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/28244b971e5b47ed9729d21d320b9e59, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dd625a5a82c44df68c289cc850f767fb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e7cbbd74a90d4b41ac54e1db41197a88] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=36.7 K 2024-12-03T02:35:38,329 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 28244b971e5b47ed9729d21d320b9e59, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733193334569 2024-12-03T02:35:38,329 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting dd625a5a82c44df68c289cc850f767fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733193335626 2024-12-03T02:35:38,330 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e7cbbd74a90d4b41ac54e1db41197a88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733193336521 2024-12-03T02:35:38,335 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:38,335 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:38,335 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:38,336 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/6b9d374ab6604a4db28296d94d22c1c0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/742d698bf1d24855a4b2531e631838d4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c62267068d9e433c8dfb434c042a743c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=36.7 K 2024-12-03T02:35:38,340 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b9d374ab6604a4db28296d94d22c1c0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733193334569 2024-12-03T02:35:38,345 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 742d698bf1d24855a4b2531e631838d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733193335626 2024-12-03T02:35:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T02:35:38,351 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting c62267068d9e433c8dfb434c042a743c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733193336521 2024-12-03T02:35:38,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T02:35:38,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:38,352 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T02:35:38,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:38,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:38,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:38,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:38,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:38,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:38,361 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:38,362 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/35bf35e763544f959d158a41936cc349 is 50, key is test_row_0/A:col10/1733193336561/Put/seqid=0 2024-12-03T02:35:38,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/b3bb2a37a37940f68384bcf88796e92a is 50, key is test_row_0/A:col10/1733193337548/Put/seqid=0 2024-12-03T02:35:38,404 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:38,405 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/b86d4fbddbdd407abea75a09e93646ac is 50, key is test_row_0/B:col10/1733193336561/Put/seqid=0 2024-12-03T02:35:38,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742323_1499 (size=13051) 2024-12-03T02:35:38,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742324_1500 (size=12301) 2024-12-03T02:35:38,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742325_1501 (size=13051) 2024-12-03T02:35:38,479 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/35bf35e763544f959d158a41936cc349 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/35bf35e763544f959d158a41936cc349 2024-12-03T02:35:38,535 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into 35bf35e763544f959d158a41936cc349(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:38,535 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:38,535 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=13, startTime=1733193338323; duration=0sec 2024-12-03T02:35:38,536 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:38,536 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:38,536 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:38,544 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:38,544 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:38,544 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:38,545 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f4908225d234803a85b9bf510c27144, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1f3fdc0af95f431e847f2fb45b49be57, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/a6f76c32d15f43b8b0d44b447f0ceaa7] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=36.7 K 2024-12-03T02:35:38,553 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f4908225d234803a85b9bf510c27144, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733193334569 2024-12-03T02:35:38,555 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3fdc0af95f431e847f2fb45b49be57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733193335626 2024-12-03T02:35:38,557 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting a6f76c32d15f43b8b0d44b447f0ceaa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733193336521 2024-12-03T02:35:38,589 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:38,590 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/4c7526db8c5641399741571fee90b0cb is 50, key is test_row_0/C:col10/1733193336561/Put/seqid=0 2024-12-03T02:35:38,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742326_1502 (size=13051) 2024-12-03T02:35:38,625 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/4c7526db8c5641399741571fee90b0cb as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c7526db8c5641399741571fee90b0cb 2024-12-03T02:35:38,633 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into 4c7526db8c5641399741571fee90b0cb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:38,633 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:38,633 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=13, startTime=1733193338323; duration=0sec 2024-12-03T02:35:38,633 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:38,633 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T02:35:38,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193398778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193398778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:38,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193398779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:38,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193398785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:38,788 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in b35fa9ee617f001aec9f096271ca1ed8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:38,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:38,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193398795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:38,826 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/b3bb2a37a37940f68384bcf88796e92a 2024-12-03T02:35:38,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/aed9f9b7705d45d1b1d4245a6bbd038c is 50, key is test_row_0/B:col10/1733193337548/Put/seqid=0 2024-12-03T02:35:38,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742327_1503 (size=12301) 2024-12-03T02:35:38,851 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/aed9f9b7705d45d1b1d4245a6bbd038c 2024-12-03T02:35:38,859 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/b86d4fbddbdd407abea75a09e93646ac as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b86d4fbddbdd407abea75a09e93646ac 2024-12-03T02:35:38,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/8d5b822b6a8b420a9ab7eddfdb47a232 is 50, key is test_row_0/C:col10/1733193337548/Put/seqid=0 2024-12-03T02:35:38,864 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into b86d4fbddbdd407abea75a09e93646ac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:38,864 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:38,865 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=13, startTime=1733193338323; duration=0sec 2024-12-03T02:35:38,865 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:38,865 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:38,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742328_1504 (size=12301) 2024-12-03T02:35:38,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193398895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:38,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193398895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:38,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193398895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193399104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193399104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193399114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T02:35:39,277 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/8d5b822b6a8b420a9ab7eddfdb47a232 2024-12-03T02:35:39,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/b3bb2a37a37940f68384bcf88796e92a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b3bb2a37a37940f68384bcf88796e92a 2024-12-03T02:35:39,292 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b3bb2a37a37940f68384bcf88796e92a, entries=150, sequenceid=491, filesize=12.0 K 2024-12-03T02:35:39,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/aed9f9b7705d45d1b1d4245a6bbd038c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/aed9f9b7705d45d1b1d4245a6bbd038c 2024-12-03T02:35:39,307 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/aed9f9b7705d45d1b1d4245a6bbd038c, entries=150, sequenceid=491, filesize=12.0 K 2024-12-03T02:35:39,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/8d5b822b6a8b420a9ab7eddfdb47a232 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8d5b822b6a8b420a9ab7eddfdb47a232 2024-12-03T02:35:39,324 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8d5b822b6a8b420a9ab7eddfdb47a232, entries=150, sequenceid=491, filesize=12.0 K 2024-12-03T02:35:39,324 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for b35fa9ee617f001aec9f096271ca1ed8 in 972ms, sequenceid=491, compaction requested=false 2024-12-03T02:35:39,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:39,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:39,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-03T02:35:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-03T02:35:39,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-03T02:35:39,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2980 sec 2024-12-03T02:35:39,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.3030 sec 2024-12-03T02:35:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:39,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T02:35:39,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:39,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:39,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:39,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:39,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:39,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:39,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/7016c8f666b6442a908f8db36670ddab is 50, key is test_row_0/A:col10/1733193338776/Put/seqid=0 2024-12-03T02:35:39,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742329_1505 (size=17181) 2024-12-03T02:35:39,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/7016c8f666b6442a908f8db36670ddab 2024-12-03T02:35:39,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/ae4b892ca74a46589a0fd5ae9551f332 is 50, key is test_row_0/B:col10/1733193338776/Put/seqid=0 2024-12-03T02:35:39,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742330_1506 (size=12301) 2024-12-03T02:35:39,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193399476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193399479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193399479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193399594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193399595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193399595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193399814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193399814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193399815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193399815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,820 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in b35fa9ee617f001aec9f096271ca1ed8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:39,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:39,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:39,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/ae4b892ca74a46589a0fd5ae9551f332 2024-12-03T02:35:39,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/cfbd1622bfa54ee9b92ab7b633b6185e is 50, key is test_row_0/C:col10/1733193338776/Put/seqid=0 2024-12-03T02:35:39,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742331_1507 (size=12301) 2024-12-03T02:35:40,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193400125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:40,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193400125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:40,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193400125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T02:35:40,185 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:40,185 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-03T02:35:40,187 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:40,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T02:35:40,187 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:40,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T02:35:40,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/cfbd1622bfa54ee9b92ab7b633b6185e 2024-12-03T02:35:40,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/7016c8f666b6442a908f8db36670ddab as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/7016c8f666b6442a908f8db36670ddab 2024-12-03T02:35:40,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/7016c8f666b6442a908f8db36670ddab, entries=250, sequenceid=507, filesize=16.8 K 2024-12-03T02:35:40,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/ae4b892ca74a46589a0fd5ae9551f332 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/ae4b892ca74a46589a0fd5ae9551f332 2024-12-03T02:35:40,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/ae4b892ca74a46589a0fd5ae9551f332, entries=150, sequenceid=507, filesize=12.0 K 2024-12-03T02:35:40,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/cfbd1622bfa54ee9b92ab7b633b6185e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/cfbd1622bfa54ee9b92ab7b633b6185e 2024-12-03T02:35:40,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/cfbd1622bfa54ee9b92ab7b633b6185e, entries=150, sequenceid=507, filesize=12.0 K 2024-12-03T02:35:40,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for b35fa9ee617f001aec9f096271ca1ed8 in 909ms, sequenceid=507, compaction requested=true 2024-12-03T02:35:40,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:40,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:40,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:40,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:40,327 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:40,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:40,327 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:40,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:40,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:40,328 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:40,328 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:40,328 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:40,328 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b86d4fbddbdd407abea75a09e93646ac, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/aed9f9b7705d45d1b1d4245a6bbd038c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/ae4b892ca74a46589a0fd5ae9551f332] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=36.8 K 2024-12-03T02:35:40,328 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:40,328 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:40,328 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:40,328 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/35bf35e763544f959d158a41936cc349, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b3bb2a37a37940f68384bcf88796e92a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/7016c8f666b6442a908f8db36670ddab] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=41.5 K 2024-12-03T02:35:40,329 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b86d4fbddbdd407abea75a09e93646ac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733193336521 2024-12-03T02:35:40,329 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35bf35e763544f959d158a41936cc349, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733193336521 2024-12-03T02:35:40,329 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3bb2a37a37940f68384bcf88796e92a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733193337548 2024-12-03T02:35:40,329 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting aed9f9b7705d45d1b1d4245a6bbd038c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733193337548 2024-12-03T02:35:40,329 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7016c8f666b6442a908f8db36670ddab, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733193338760 2024-12-03T02:35:40,329 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ae4b892ca74a46589a0fd5ae9551f332, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733193338776 2024-12-03T02:35:40,334 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:40,334 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/0f557b62d265404ea6653dd8ea72ac8a is 50, key is test_row_0/A:col10/1733193338776/Put/seqid=0 2024-12-03T02:35:40,334 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:40,335 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/436f30333d4f4bd4be29a339bf70ad55 is 50, key is test_row_0/B:col10/1733193338776/Put/seqid=0 2024-12-03T02:35:40,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T02:35:40,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:40,339 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T02:35:40,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:40,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:40,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:40,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:40,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:40,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:40,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742332_1508 (size=13153) 2024-12-03T02:35:40,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742333_1509 (size=13153) 2024-12-03T02:35:40,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/561227140e204bb99ab493456a502c7d is 50, key is test_row_0/A:col10/1733193339476/Put/seqid=0 2024-12-03T02:35:40,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742334_1510 (size=12301) 2024-12-03T02:35:40,377 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/561227140e204bb99ab493456a502c7d 2024-12-03T02:35:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/0931183511f540dcb03a5fb3bf398730 is 50, key is test_row_0/B:col10/1733193339476/Put/seqid=0 2024-12-03T02:35:40,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742335_1511 (size=12301) 2024-12-03T02:35:40,395 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/0931183511f540dcb03a5fb3bf398730 2024-12-03T02:35:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/8a75aece25534afdb444bf67f2b820a4 is 50, key is test_row_0/C:col10/1733193339476/Put/seqid=0 2024-12-03T02:35:40,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742336_1512 (size=12301) 2024-12-03T02:35:40,424 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/8a75aece25534afdb444bf67f2b820a4 2024-12-03T02:35:40,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/561227140e204bb99ab493456a502c7d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/561227140e204bb99ab493456a502c7d 2024-12-03T02:35:40,431 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/561227140e204bb99ab493456a502c7d, entries=150, sequenceid=527, filesize=12.0 K 2024-12-03T02:35:40,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/0931183511f540dcb03a5fb3bf398730 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0931183511f540dcb03a5fb3bf398730 2024-12-03T02:35:40,437 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0931183511f540dcb03a5fb3bf398730, entries=150, sequenceid=527, filesize=12.0 K 2024-12-03T02:35:40,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/8a75aece25534afdb444bf67f2b820a4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8a75aece25534afdb444bf67f2b820a4 2024-12-03T02:35:40,442 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8a75aece25534afdb444bf67f2b820a4, entries=150, sequenceid=527, filesize=12.0 K 2024-12-03T02:35:40,442 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for b35fa9ee617f001aec9f096271ca1ed8 in 103ms, sequenceid=527, compaction requested=true 2024-12-03T02:35:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-03T02:35:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-03T02:35:40,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-03T02:35:40,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 257 msec 2024-12-03T02:35:40,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 260 msec 2024-12-03T02:35:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T02:35:40,505 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:40,505 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-03T02:35:40,507 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T02:35:40,508 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:40,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T02:35:40,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T02:35:40,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:40,661 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T02:35:40,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:40,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:40,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/fd5cebb5b1ad4ca5985b6a9eb585654e is 50, key is test_row_0/A:col10/1733193340648/Put/seqid=0 2024-12-03T02:35:40,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742337_1513 (size=17181) 2024-12-03T02:35:40,683 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/fd5cebb5b1ad4ca5985b6a9eb585654e 2024-12-03T02:35:40,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/237e2168f3b04a34869cdf551de97e37 is 50, key is test_row_0/B:col10/1733193340648/Put/seqid=0 2024-12-03T02:35:40,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742338_1514 (size=12301) 2024-12-03T02:35:40,706 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/237e2168f3b04a34869cdf551de97e37 2024-12-03T02:35:40,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/223e231d355b45d680c8b63bb66797b4 is 50, key is test_row_0/C:col10/1733193340648/Put/seqid=0 2024-12-03T02:35:40,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742339_1515 (size=12301) 2024-12-03T02:35:40,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193340714, max timestamp=1733193340736Num uniques 859; 2024-12-03T02:35:40,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193340714, max timestamp=1733193340736Num uniques 859; 2024-12-03T02:35:40,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:35:40,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:35:40,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193340714, max timestamp=1733193340736Num uniques 859; 2024-12-03T02:35:40,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:35:40,747 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/0f557b62d265404ea6653dd8ea72ac8a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0f557b62d265404ea6653dd8ea72ac8a 2024-12-03T02:35:40,750 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/436f30333d4f4bd4be29a339bf70ad55 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/436f30333d4f4bd4be29a339bf70ad55 2024-12-03T02:35:40,753 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into 0f557b62d265404ea6653dd8ea72ac8a(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:40,753 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:40,753 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=13, startTime=1733193340327; duration=0sec 2024-12-03T02:35:40,753 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:40,753 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:40,753 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:40,754 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into 436f30333d4f4bd4be29a339bf70ad55(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:40,754 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:40,754 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=13, startTime=1733193340327; duration=0sec 2024-12-03T02:35:40,754 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:40,754 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:40,754 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:40,754 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:40,754 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:40,754 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c7526db8c5641399741571fee90b0cb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8d5b822b6a8b420a9ab7eddfdb47a232, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/cfbd1622bfa54ee9b92ab7b633b6185e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8a75aece25534afdb444bf67f2b820a4] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=48.8 K 2024-12-03T02:35:40,754 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c7526db8c5641399741571fee90b0cb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733193336521 2024-12-03T02:35:40,754 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d5b822b6a8b420a9ab7eddfdb47a232, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733193337548 2024-12-03T02:35:40,755 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfbd1622bfa54ee9b92ab7b633b6185e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733193338776 2024-12-03T02:35:40,755 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a75aece25534afdb444bf67f2b820a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733193339461 2024-12-03T02:35:40,763 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:40,764 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/0fc22cbac0a249e1ab8faf2069c7c143 is 50, key is test_row_0/C:col10/1733193339476/Put/seqid=0 2024-12-03T02:35:40,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742340_1516 (size=13187) 2024-12-03T02:35:40,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193400796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:40,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193340777, max timestamp=1733193340811Num uniques 1018; 2024-12-03T02:35:40,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193340777, max timestamp=1733193340811Num uniques 1018; 2024-12-03T02:35:40,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T02:35:40,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T02:35:40,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193340777, max timestamp=1733193340811Num uniques 1018; 2024-12-03T02:35:40,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T02:35:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T02:35:40,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733193340880, max timestamp=1733193340902Num uniques 1077; 2024-12-03T02:35:40,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733193340880, max timestamp=1733193340902Num uniques 1077; 2024-12-03T02:35:40,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T02:35:40,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T02:35:40,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733193340880, max timestamp=1733193340902Num uniques 1077; 2024-12-03T02:35:40,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T02:35:40,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193400976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:40,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:41,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193400987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:41,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193400987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:41,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193401095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:41,117 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/223e231d355b45d680c8b63bb66797b4 2024-12-03T02:35:41,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:41,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193401114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:41,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193401114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:41,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/fd5cebb5b1ad4ca5985b6a9eb585654e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd5cebb5b1ad4ca5985b6a9eb585654e 2024-12-03T02:35:41,129 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd5cebb5b1ad4ca5985b6a9eb585654e, entries=250, sequenceid=536, filesize=16.8 K 2024-12-03T02:35:41,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/237e2168f3b04a34869cdf551de97e37 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/237e2168f3b04a34869cdf551de97e37 2024-12-03T02:35:41,133 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/237e2168f3b04a34869cdf551de97e37, entries=150, sequenceid=536, filesize=12.0 K 2024-12-03T02:35:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T02:35:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/223e231d355b45d680c8b63bb66797b4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/223e231d355b45d680c8b63bb66797b4 2024-12-03T02:35:41,138 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/223e231d355b45d680c8b63bb66797b4, entries=150, sequenceid=536, filesize=12.0 K 2024-12-03T02:35:41,141 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for b35fa9ee617f001aec9f096271ca1ed8 in 479ms, sequenceid=536, compaction requested=true 2024-12-03T02:35:41,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:41,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:41,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-03T02:35:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-03T02:35:41,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-03T02:35:41,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-12-03T02:35:41,149 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 643 msec 2024-12-03T02:35:41,172 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/0fc22cbac0a249e1ab8faf2069c7c143 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/0fc22cbac0a249e1ab8faf2069c7c143 2024-12-03T02:35:41,195 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into 0fc22cbac0a249e1ab8faf2069c7c143(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:41,195 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:41,195 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=12, startTime=1733193340327; duration=0sec 2024-12-03T02:35:41,195 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:41,195 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:41,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733193340974, max timestamp=1733193341309Num uniques 1086; 2024-12-03T02:35:41,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-03T02:35:41,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733193340974, max timestamp=1733193341309Num uniques 1086; 2024-12-03T02:35:41,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-03T02:35:41,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=67.09 KB heapSize=182.19 KB 2024-12-03T02:35:41,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:41,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:41,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733193340974, max timestamp=1733193341309Num uniques 1086; 2024-12-03T02:35:41,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:41,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:41,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-03T02:35:41,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:41,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:41,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/fd167f8948484e6e841e9e28448a71f4 is 50, key is test_row_0/A:col10/1733193340984/Put/seqid=0 2024-12-03T02:35:41,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742341_1517 (size=14741) 2024-12-03T02:35:41,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193341352, max timestamp=1733193341380Num uniques 859; 2024-12-03T02:35:41,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:41,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193341352, max timestamp=1733193341380Num uniques 859; 2024-12-03T02:35:41,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733193341352, max timestamp=1733193341380Num uniques 859; 2024-12-03T02:35:41,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:35:41,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T02:35:41,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193341417, max timestamp=1733193341438Num uniques 1118; 2024-12-03T02:35:41,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-03T02:35:41,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193341417, max timestamp=1733193341438Num uniques 1118; 2024-12-03T02:35:41,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-03T02:35:41,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733193341417, max timestamp=1733193341438Num uniques 1068; 2024-12-03T02:35:41,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-03T02:35:41,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733193341487, max timestamp=1733193341506Num uniques 1077; 2024-12-03T02:35:41,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T02:35:41,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733193341487, max timestamp=1733193341506Num uniques 1077; 2024-12-03T02:35:41,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T02:35:41,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733193341487, max timestamp=1733193341506Num uniques 1077; 2024-12-03T02:35:41,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T02:35:41,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733193341569, max timestamp=1733193341607Num uniques 1086; 2024-12-03T02:35:41,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733193341569, max timestamp=1733193341607Num uniques 1086; 2024-12-03T02:35:41,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T02:35:41,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T02:35:41,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733193341569, max timestamp=1733193341607Num uniques 1086; 2024-12-03T02:35:41,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T02:35:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T02:35:41,644 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:41,651 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-03T02:35:41,653 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:35:41,653 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:41,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:41,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733193341653, max timestamp=1733193341670Num uniques 1095; 2024-12-03T02:35:41,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-03T02:35:41,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733193341653, max timestamp=1733193341670Num uniques 1095; 2024-12-03T02:35:41,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-03T02:35:41,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733193341653, max timestamp=1733193341670Num uniques 1095; 2024-12-03T02:35:41,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-03T02:35:41,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/fd167f8948484e6e841e9e28448a71f4 2024-12-03T02:35:41,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/c4c36b425ee24ac38d86754c77d05304 is 50, key is test_row_0/B:col10/1733193340984/Put/seqid=0 2024-12-03T02:35:41,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733193341710, max timestamp=1733193341737Num uniques 1154; 2024-12-03T02:35:41,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733193341710, max timestamp=1733193341737Num uniques 1154; 2024-12-03T02:35:41,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-03T02:35:41,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-03T02:35:41,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733193341710, max timestamp=1733193341737Num uniques 1154; 2024-12-03T02:35:41,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-03T02:35:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:35:41,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742342_1518 (size=12301) 2024-12-03T02:35:41,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:41,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:41,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:41,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:41,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:41,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:41,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733193341773, max timestamp=1733193341804Num uniques 1013; 2024-12-03T02:35:41,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733193341773, max timestamp=1733193341804Num uniques 1013; 2024-12-03T02:35:41,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-03T02:35:41,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-03T02:35:41,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733193341773, max timestamp=1733193341804Num uniques 1013; 2024-12-03T02:35:41,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-03T02:35:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733193341850, max timestamp=1733193341868Num uniques 1022; 2024-12-03T02:35:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-12-03T02:35:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733193341850, max timestamp=1733193341868Num uniques 1022; 2024-12-03T02:35:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733193341850, max timestamp=1733193341868Num uniques 1022; 2024-12-03T02:35:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-12-03T02:35:41,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-12-03T02:35:41,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733193341936, max timestamp=1733193341953Num uniques 1131; 2024-12-03T02:35:41,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-03T02:35:41,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733193341936, max timestamp=1733193341953Num uniques 1131; 2024-12-03T02:35:41,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-03T02:35:41,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733193341936, max timestamp=1733193341953Num uniques 1131; 2024-12-03T02:35:41,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=8286, suffix heap size=145592,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=181 2024-12-03T02:35:41,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:41,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:41,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:41,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:41,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:41,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:35:42,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733193341985, max timestamp=1733193342005Num uniques 1090; 2024-12-03T02:35:42,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-03T02:35:42,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733193341985, max timestamp=1733193342005Num uniques 1040; 2024-12-03T02:35:42,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733193341985, max timestamp=1733193342005Num uniques 1090; 2024-12-03T02:35:42,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-03T02:35:42,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-03T02:35:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733193342045, max timestamp=1733193342061Num uniques 1099; 2024-12-03T02:35:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-12-03T02:35:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733193342045, max timestamp=1733193342061Num uniques 1099; 2024-12-03T02:35:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-12-03T02:35:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733193342045, max timestamp=1733193342061Num uniques 1099; 2024-12-03T02:35:42,065 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-12-03T02:35:42,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:42,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:42,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733193342096, max timestamp=1733193342125Num uniques 1158; 2024-12-03T02:35:42,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-12-03T02:35:42,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733193342096, max timestamp=1733193342125Num uniques 1158; 2024-12-03T02:35:42,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-12-03T02:35:42,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733193342096, max timestamp=1733193342125Num uniques 1158; 2024-12-03T02:35:42,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-12-03T02:35:42,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/c4c36b425ee24ac38d86754c77d05304 2024-12-03T02:35:42,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=317, cellSize=14512, totalHeapSize=33208, min timestamp=1733193342156, max timestamp=1733193342179Num uniques 1017; 2024-12-03T02:35:42,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=14512, suffix heap size=133736,new segment heap size=33208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=317 2024-12-03T02:35:42,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=317, cellSize=14512, totalHeapSize=33208, min timestamp=1733193342156, max timestamp=1733193342179Num uniques 1017; 2024-12-03T02:35:42,183 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=14512, suffix heap size=133736,new segment heap size=33208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=317 2024-12-03T02:35:42,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=317, cellSize=14512, totalHeapSize=33208, min timestamp=1733193342156, max timestamp=1733193342179Num uniques 1017; 2024-12-03T02:35:42,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=14512, suffix heap size=133736,new segment heap size=33208 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=317 2024-12-03T02:35:42,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/6c2cf25818e14e76adbf11db1a8a492c is 50, key is test_row_0/C:col10/1733193340984/Put/seqid=0 2024-12-03T02:35:42,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742343_1519 (size=12301) 2024-12-03T02:35:42,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733193342208, max timestamp=1733193342231Num uniques 1176; 2024-12-03T02:35:42,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53856, new segment data size=10346, suffix heap size=150272,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1176, new segment cells count=226 2024-12-03T02:35:42,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733193342208, max timestamp=1733193342231Num uniques 1176; 2024-12-03T02:35:42,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53856, new segment data size=10346, suffix heap size=150272,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1176, new segment cells count=226 2024-12-03T02:35:42,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733193342208, max timestamp=1733193342231Num uniques 1176; 2024-12-03T02:35:42,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53856, new segment data size=10346, suffix heap size=150272,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1176, new segment cells count=226 2024-12-03T02:35:42,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:35:42,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733193342270, max timestamp=1733193342297Num uniques 1085; 2024-12-03T02:35:42,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-03T02:35:42,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733193342270, max timestamp=1733193342297Num uniques 1085; 2024-12-03T02:35:42,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-03T02:35:42,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733193342270, max timestamp=1733193342297Num uniques 1085; 2024-12-03T02:35:42,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=8470, suffix heap size=140808,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=185 2024-12-03T02:35:42,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733193342348, max timestamp=1733193342371Num uniques 1044; 2024-12-03T02:35:42,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-03T02:35:42,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733193342348, max timestamp=1733193342371Num uniques 1044; 2024-12-03T02:35:42,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-03T02:35:42,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1733193342348, max timestamp=1733193342371Num uniques 1044; 2024-12-03T02:35:42,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8884, suffix heap size=136544,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=194 2024-12-03T02:35:42,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733193342412, max timestamp=1733193342435Num uniques 1053; 2024-12-03T02:35:42,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-12-03T02:35:42,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733193342412, max timestamp=1733193342435Num uniques 1053; 2024-12-03T02:35:42,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-12-03T02:35:42,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1733193342412, max timestamp=1733193342435Num uniques 1053; 2024-12-03T02:35:42,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-12-03T02:35:42,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1733193342454, max timestamp=1733193342479Num uniques 1012; 2024-12-03T02:35:42,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=11992, suffix heap size=133216,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=262 2024-12-03T02:35:42,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1733193342454, max timestamp=1733193342479Num uniques 1012; 2024-12-03T02:35:42,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=262, cellSize=11992, totalHeapSize=27488, min timestamp=1733193342454, max timestamp=1733193342479Num uniques 1012; 2024-12-03T02:35:42,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=11992, suffix heap size=133216,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=262 2024-12-03T02:35:42,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=11992, suffix heap size=133216,new segment heap size=27488 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=262 2024-12-03T02:35:42,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1733193342516, max timestamp=1733193342531Num uniques 1121; 2024-12-03T02:35:42,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=10116, suffix heap size=144552,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=221 2024-12-03T02:35:42,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1733193342516, max timestamp=1733193342531Num uniques 1121; 2024-12-03T02:35:42,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=10116, suffix heap size=144552,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=221 2024-12-03T02:35:42,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1733193342517, max timestamp=1733193342531Num uniques 1121; 2024-12-03T02:35:42,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51336, new segment data size=7826, suffix heap size=144552,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1121, new segment cells count=171 2024-12-03T02:35:42,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:42,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:42,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/6c2cf25818e14e76adbf11db1a8a492c 2024-12-03T02:35:42,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/fd167f8948484e6e841e9e28448a71f4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd167f8948484e6e841e9e28448a71f4 2024-12-03T02:35:42,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1733193342566, max timestamp=1733193342596Num uniques 1080; 2024-12-03T02:35:42,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-12-03T02:35:42,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1733193342566, max timestamp=1733193342596Num uniques 1080; 2024-12-03T02:35:42,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-12-03T02:35:42,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1733193342566, max timestamp=1733193342596Num uniques 1030; 2024-12-03T02:35:42,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47170, new segment data size=10530, suffix heap size=135088,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=230 2024-12-03T02:35:42,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd167f8948484e6e841e9e28448a71f4, entries=200, sequenceid=611, filesize=14.4 K 2024-12-03T02:35:42,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/c4c36b425ee24ac38d86754c77d05304 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c4c36b425ee24ac38d86754c77d05304 2024-12-03T02:35:42,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c4c36b425ee24ac38d86754c77d05304, entries=150, sequenceid=611, filesize=12.0 K 2024-12-03T02:35:42,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/6c2cf25818e14e76adbf11db1a8a492c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/6c2cf25818e14e76adbf11db1a8a492c 2024-12-03T02:35:42,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/6c2cf25818e14e76adbf11db1a8a492c, entries=150, sequenceid=611, filesize=12.0 K 2024-12-03T02:35:42,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~63.66 KB/65184, currentSize=53.67 KB/54960 for b35fa9ee617f001aec9f096271ca1ed8 in 1307ms, sequenceid=611, compaction requested=true 2024-12-03T02:35:42,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:42,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:42,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:42,620 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:42,620 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:42,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:42,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:42,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:42,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:42,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=73.80 KB heapSize=199.03 KB 2024-12-03T02:35:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:42,621 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:42,621 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57376 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:42,622 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:42,622 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:42,622 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,622 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0f557b62d265404ea6653dd8ea72ac8a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/561227140e204bb99ab493456a502c7d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd5cebb5b1ad4ca5985b6a9eb585654e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd167f8948484e6e841e9e28448a71f4] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=56.0 K 2024-12-03T02:35:42,622 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,622 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/436f30333d4f4bd4be29a339bf70ad55, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0931183511f540dcb03a5fb3bf398730, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/237e2168f3b04a34869cdf551de97e37, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c4c36b425ee24ac38d86754c77d05304] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=48.9 K 2024-12-03T02:35:42,622 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 436f30333d4f4bd4be29a339bf70ad55, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733193338776 2024-12-03T02:35:42,622 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f557b62d265404ea6653dd8ea72ac8a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1733193338776 2024-12-03T02:35:42,623 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 561227140e204bb99ab493456a502c7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733193339461 2024-12-03T02:35:42,623 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 0931183511f540dcb03a5fb3bf398730, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733193339461 2024-12-03T02:35:42,623 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd5cebb5b1ad4ca5985b6a9eb585654e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1733193340648 2024-12-03T02:35:42,623 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 237e2168f3b04a34869cdf551de97e37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1733193340648 2024-12-03T02:35:42,625 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd167f8948484e6e841e9e28448a71f4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733193340974 2024-12-03T02:35:42,625 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c36b425ee24ac38d86754c77d05304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733193340983 2024-12-03T02:35:42,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/dfdff82d0dfc4c2396c466a98bea9908 is 50, key is test_row_0/A:col10/1733193342618/Put/seqid=0 2024-12-03T02:35:42,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742344_1520 (size=12301) 2024-12-03T02:35:42,641 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:42,642 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/5ff1a81e81fd49659b71bedd16227912 is 50, key is test_row_0/A:col10/1733193340984/Put/seqid=0 2024-12-03T02:35:42,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=964 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/dfdff82d0dfc4c2396c466a98bea9908 2024-12-03T02:35:42,659 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:42,659 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/e4531c9c9bef41e5ba0715c989e6e7a7 is 50, key is test_row_0/B:col10/1733193340984/Put/seqid=0 2024-12-03T02:35:42,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/7d0dc2a3282946be8055b8728c90457e is 50, key is test_row_0/B:col10/1733193342618/Put/seqid=0 2024-12-03T02:35:42,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742345_1521 (size=13289) 2024-12-03T02:35:42,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193402674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:42,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193402680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:42,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193402704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:42,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193402704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:42,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:42,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:42,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742346_1522 (size=13289) 2024-12-03T02:35:42,722 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/e4531c9c9bef41e5ba0715c989e6e7a7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/e4531c9c9bef41e5ba0715c989e6e7a7 2024-12-03T02:35:42,726 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into e4531c9c9bef41e5ba0715c989e6e7a7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:42,727 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:42,727 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=12, startTime=1733193342620; duration=0sec 2024-12-03T02:35:42,727 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:42,727 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:42,727 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:35:42,728 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:35:42,728 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:42,728 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,728 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/0fc22cbac0a249e1ab8faf2069c7c143, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/223e231d355b45d680c8b63bb66797b4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/6c2cf25818e14e76adbf11db1a8a492c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=36.9 K 2024-12-03T02:35:42,728 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc22cbac0a249e1ab8faf2069c7c143, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733193339461 2024-12-03T02:35:42,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:42,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742347_1523 (size=12301) 2024-12-03T02:35:42,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:42,731 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 223e231d355b45d680c8b63bb66797b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=536, earliestPutTs=1733193340648 2024-12-03T02:35:42,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,731 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c2cf25818e14e76adbf11db1a8a492c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733193340983 2024-12-03T02:35:42,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,745 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:42,746 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/c9c602157cb143169a4bd4bbb9a49bb9 is 50, key is test_row_0/C:col10/1733193340984/Put/seqid=0 2024-12-03T02:35:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742348_1524 (size=13289) 2024-12-03T02:35:42,787 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/c9c602157cb143169a4bd4bbb9a49bb9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/c9c602157cb143169a4bd4bbb9a49bb9 2024-12-03T02:35:42,791 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into c9c602157cb143169a4bd4bbb9a49bb9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:42,791 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:42,791 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=13, startTime=1733193342620; duration=0sec 2024-12-03T02:35:42,791 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:42,791 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:35:42,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193402808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:42,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:42,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193402808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58662 deadline: 1733193402815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:42,829 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_1' in b35fa9ee617f001aec9f096271ca1ed8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:35:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:42,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193402824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:42,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193402824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:42,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:42,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:42,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:42,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:43,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:43,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:43,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:43,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193403035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193403035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193403045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193403045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,098 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/5ff1a81e81fd49659b71bedd16227912 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5ff1a81e81fd49659b71bedd16227912 2024-12-03T02:35:43,102 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into 5ff1a81e81fd49659b71bedd16227912(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:43,102 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:43,102 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=12, startTime=1733193342620; duration=0sec 2024-12-03T02:35:43,102 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:43,102 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:43,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=964 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/7d0dc2a3282946be8055b8728c90457e 2024-12-03T02:35:43,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/e597d2736cc74423863b75751a0e49ce is 50, key is test_row_0/C:col10/1733193342618/Put/seqid=0 2024-12-03T02:35:43,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742349_1525 (size=12301) 2024-12-03T02:35:43,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:43,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:43,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:43,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:43,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:43,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:43,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193403355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193403355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193403364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193403364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:43,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:43,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=964 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/e597d2736cc74423863b75751a0e49ce 2024-12-03T02:35:43,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/dfdff82d0dfc4c2396c466a98bea9908 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dfdff82d0dfc4c2396c466a98bea9908 2024-12-03T02:35:43,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dfdff82d0dfc4c2396c466a98bea9908, entries=150, sequenceid=964, filesize=12.0 K 2024-12-03T02:35:43,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/7d0dc2a3282946be8055b8728c90457e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d0dc2a3282946be8055b8728c90457e 2024-12-03T02:35:43,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d0dc2a3282946be8055b8728c90457e, entries=150, sequenceid=964, filesize=12.0 K 2024-12-03T02:35:43,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/e597d2736cc74423863b75751a0e49ce as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/e597d2736cc74423863b75751a0e49ce 2024-12-03T02:35:43,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/e597d2736cc74423863b75751a0e49ce, entries=150, sequenceid=964, filesize=12.0 K 2024-12-03T02:35:43,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~198.98 KB/203760, currentSize=107.34 KB/109920 for b35fa9ee617f001aec9f096271ca1ed8 in 1027ms, sequenceid=964, compaction requested=false 2024-12-03T02:35:43,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:43,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T02:35:43,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:43,664 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T02:35:43,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:43,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:43,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:43,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:43,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:43,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:43,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/99e64d0794c84cc18298c6f6d7e9c341 is 50, key is test_row_0/A:col10/1733193342675/Put/seqid=0 2024-12-03T02:35:43,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742350_1526 (size=12301) 2024-12-03T02:35:43,697 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=986 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/99e64d0794c84cc18298c6f6d7e9c341 2024-12-03T02:35:43,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/5ddd1750bea24d1285d6b6b8f4a8e9cb is 50, key is test_row_0/B:col10/1733193342675/Put/seqid=0 2024-12-03T02:35:43,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742351_1527 (size=12301) 2024-12-03T02:35:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:35:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:43,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. as already flushing 2024-12-03T02:35:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193403911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193403912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193403912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193403912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:43,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:43,936 INFO [Thread-1959 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1959. 2024-12-03T02:35:43,936 DEBUG [Thread-1959 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:43,936 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,936 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:43,938 INFO [Thread-1953 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1953. 2024-12-03T02:35:43,938 DEBUG [Thread-1953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:43,938 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,938 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:43,940 INFO [Thread-1957 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1957. 2024-12-03T02:35:43,940 DEBUG [Thread-1957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:43,940 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,941 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:43,942 INFO [Thread-1961 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1961. 2024-12-03T02:35:43,942 DEBUG [Thread-1961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:43,942 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,942 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:43,946 INFO [Thread-1955 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1955. 2024-12-03T02:35:43,946 DEBUG [Thread-1955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:43,946 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,946 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:43,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:44,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:44,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:44,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58708 deadline: 1733193404035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58706 deadline: 1733193404035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58680 deadline: 1733193404035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:44,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58674 deadline: 1733193404035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 , the old value is region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:44,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., hostname=64035cbb0b46,38721,1733193202350, seqNum=2 because the exception is null or not the one we care about 2024-12-03T02:35:44,122 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=986 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/5ddd1750bea24d1285d6b6b8f4a8e9cb 2024-12-03T02:35:44,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/1a700fd0ccdf46c6883bc4ba4e7b8fad is 50, key is test_row_0/C:col10/1733193342675/Put/seqid=0 2024-12-03T02:35:44,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742352_1528 (size=12301) 2024-12-03T02:35:44,133 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=986 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/1a700fd0ccdf46c6883bc4ba4e7b8fad 2024-12-03T02:35:44,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/99e64d0794c84cc18298c6f6d7e9c341 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/99e64d0794c84cc18298c6f6d7e9c341 2024-12-03T02:35:44,140 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/99e64d0794c84cc18298c6f6d7e9c341, entries=150, sequenceid=986, filesize=12.0 K 2024-12-03T02:35:44,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/5ddd1750bea24d1285d6b6b8f4a8e9cb as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/5ddd1750bea24d1285d6b6b8f4a8e9cb 2024-12-03T02:35:44,145 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/5ddd1750bea24d1285d6b6b8f4a8e9cb, entries=150, sequenceid=986, filesize=12.0 K 2024-12-03T02:35:44,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/1a700fd0ccdf46c6883bc4ba4e7b8fad as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1a700fd0ccdf46c6883bc4ba4e7b8fad 2024-12-03T02:35:44,149 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1a700fd0ccdf46c6883bc4ba4e7b8fad, entries=150, sequenceid=986, filesize=12.0 K 2024-12-03T02:35:44,150 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for b35fa9ee617f001aec9f096271ca1ed8 in 487ms, sequenceid=986, compaction requested=true 2024-12-03T02:35:44,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:44,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:44,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-03T02:35:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-03T02:35:44,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-03T02:35:44,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4990 sec 2024-12-03T02:35:44,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.5030 sec 2024-12-03T02:35:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:44,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T02:35:44,246 INFO [Thread-1948 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1948. 2024-12-03T02:35:44,246 DEBUG [Thread-1948 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:44,246 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:44,246 INFO [Thread-1942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1942. 2024-12-03T02:35:44,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:44,246 INFO [Thread-1950 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1950. 2024-12-03T02:35:44,246 DEBUG [Thread-1942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:44,246 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:44,246 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:44,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:44,246 DEBUG [Thread-1950 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:44,246 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:44,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:44,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:44,246 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:44,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:44,246 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:44,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:44,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:44,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:44,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:44,247 INFO [Thread-1946 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1946. 2024-12-03T02:35:44,247 DEBUG [Thread-1946 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:44,247 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:44,247 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:44,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:44,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/d26e3baaad9d4b868672e913a48e3bfa is 50, key is test_row_0/A:col10/1733193343910/Put/seqid=0 2024-12-03T02:35:44,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742353_1529 (size=12301) 2024-12-03T02:35:44,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=1003 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/d26e3baaad9d4b868672e913a48e3bfa 2024-12-03T02:35:44,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/76d244dd2a3341f3bbdc70d326419c9b is 50, key is test_row_0/B:col10/1733193343910/Put/seqid=0 2024-12-03T02:35:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742354_1530 (size=12301) 2024-12-03T02:35:45,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=1003 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/76d244dd2a3341f3bbdc70d326419c9b 2024-12-03T02:35:45,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/00b500de60c340648be0aedad6269be3 is 50, key is test_row_0/C:col10/1733193343910/Put/seqid=0 2024-12-03T02:35:45,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742355_1531 (size=12301) 2024-12-03T02:35:45,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=1003 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/00b500de60c340648be0aedad6269be3 2024-12-03T02:35:45,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/d26e3baaad9d4b868672e913a48e3bfa as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/d26e3baaad9d4b868672e913a48e3bfa 2024-12-03T02:35:45,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/d26e3baaad9d4b868672e913a48e3bfa, entries=150, sequenceid=1003, filesize=12.0 K 2024-12-03T02:35:45,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/76d244dd2a3341f3bbdc70d326419c9b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/76d244dd2a3341f3bbdc70d326419c9b 2024-12-03T02:35:45,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/76d244dd2a3341f3bbdc70d326419c9b, entries=150, sequenceid=1003, filesize=12.0 K 2024-12-03T02:35:45,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/00b500de60c340648be0aedad6269be3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/00b500de60c340648be0aedad6269be3 2024-12-03T02:35:45,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/00b500de60c340648be0aedad6269be3, entries=150, sequenceid=1003, filesize=12.0 K 2024-12-03T02:35:45,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for b35fa9ee617f001aec9f096271ca1ed8 in 1248ms, sequenceid=1003, compaction requested=true 2024-12-03T02:35:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:35:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:45,494 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:35:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35fa9ee617f001aec9f096271ca1ed8:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:35:45,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:45,494 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:45,494 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:45,494 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:45,494 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/A is initiating minor compaction (all files) 2024-12-03T02:35:45,495 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/B is initiating minor compaction (all files) 2024-12-03T02:35:45,495 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/A in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:45,495 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/B in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:45,495 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5ff1a81e81fd49659b71bedd16227912, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dfdff82d0dfc4c2396c466a98bea9908, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/99e64d0794c84cc18298c6f6d7e9c341, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/d26e3baaad9d4b868672e913a48e3bfa] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=49.0 K 2024-12-03T02:35:45,495 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/e4531c9c9bef41e5ba0715c989e6e7a7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d0dc2a3282946be8055b8728c90457e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/5ddd1750bea24d1285d6b6b8f4a8e9cb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/76d244dd2a3341f3bbdc70d326419c9b] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=49.0 K 2024-12-03T02:35:45,495 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e4531c9c9bef41e5ba0715c989e6e7a7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733193340983 2024-12-03T02:35:45,495 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ff1a81e81fd49659b71bedd16227912, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733193340983 2024-12-03T02:35:45,495 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d0dc2a3282946be8055b8728c90457e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=964, earliestPutTs=1733193342598 2024-12-03T02:35:45,495 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfdff82d0dfc4c2396c466a98bea9908, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=964, earliestPutTs=1733193342598 2024-12-03T02:35:45,495 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ddd1750bea24d1285d6b6b8f4a8e9cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=986, earliestPutTs=1733193342675 2024-12-03T02:35:45,495 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99e64d0794c84cc18298c6f6d7e9c341, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=986, earliestPutTs=1733193342675 2024-12-03T02:35:45,495 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d244dd2a3341f3bbdc70d326419c9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1003, earliestPutTs=1733193343910 2024-12-03T02:35:45,495 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting d26e3baaad9d4b868672e913a48e3bfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1003, earliestPutTs=1733193343910 2024-12-03T02:35:45,502 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#A#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:45,502 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#B#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:45,502 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/a47e5ce5a6394c4b8113a19fb26cd33b is 50, key is test_row_0/A:col10/1733193343910/Put/seqid=0 2024-12-03T02:35:45,502 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/426740060f8744b4ab4cea4354d2457e is 50, key is test_row_0/B:col10/1733193343910/Put/seqid=0 2024-12-03T02:35:45,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742356_1532 (size=13425) 2024-12-03T02:35:45,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742357_1533 (size=13425) 2024-12-03T02:35:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:35:45,825 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:45,909 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/426740060f8744b4ab4cea4354d2457e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/426740060f8744b4ab4cea4354d2457e 2024-12-03T02:35:45,909 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/a47e5ce5a6394c4b8113a19fb26cd33b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/a47e5ce5a6394c4b8113a19fb26cd33b 2024-12-03T02:35:45,913 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/B of b35fa9ee617f001aec9f096271ca1ed8 into 426740060f8744b4ab4cea4354d2457e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:45,913 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/A of b35fa9ee617f001aec9f096271ca1ed8 into a47e5ce5a6394c4b8113a19fb26cd33b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:45,913 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:45,913 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:45,913 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/B, priority=12, startTime=1733193345494; duration=0sec 2024-12-03T02:35:45,913 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/A, priority=12, startTime=1733193345494; duration=0sec 2024-12-03T02:35:45,913 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:45,913 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:35:45,913 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:A 2024-12-03T02:35:45,913 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:B 2024-12-03T02:35:45,913 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:35:45,914 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:35:45,914 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): b35fa9ee617f001aec9f096271ca1ed8/C is initiating minor compaction (all files) 2024-12-03T02:35:45,914 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35fa9ee617f001aec9f096271ca1ed8/C in TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:45,914 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/c9c602157cb143169a4bd4bbb9a49bb9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/e597d2736cc74423863b75751a0e49ce, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1a700fd0ccdf46c6883bc4ba4e7b8fad, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/00b500de60c340648be0aedad6269be3] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp, totalSize=49.0 K 2024-12-03T02:35:45,915 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c602157cb143169a4bd4bbb9a49bb9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733193340983 2024-12-03T02:35:45,915 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e597d2736cc74423863b75751a0e49ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=964, earliestPutTs=1733193342598 2024-12-03T02:35:45,915 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a700fd0ccdf46c6883bc4ba4e7b8fad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=986, earliestPutTs=1733193342675 2024-12-03T02:35:45,916 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b500de60c340648be0aedad6269be3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1003, earliestPutTs=1733193343910 2024-12-03T02:35:45,924 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35fa9ee617f001aec9f096271ca1ed8#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:35:45,924 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/870ffcbe35cf47a6bae2f63e5ce68083 is 50, key is test_row_0/C:col10/1733193343910/Put/seqid=0 2024-12-03T02:35:45,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742358_1534 (size=13425) 2024-12-03T02:35:46,338 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/870ffcbe35cf47a6bae2f63e5ce68083 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/870ffcbe35cf47a6bae2f63e5ce68083 2024-12-03T02:35:46,342 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35fa9ee617f001aec9f096271ca1ed8/C of b35fa9ee617f001aec9f096271ca1ed8 into 870ffcbe35cf47a6bae2f63e5ce68083(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:35:46,342 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35fa9ee617f001aec9f096271ca1ed8: 2024-12-03T02:35:46,342 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., storeName=b35fa9ee617f001aec9f096271ca1ed8/C, priority=12, startTime=1733193345494; duration=0sec 2024-12-03T02:35:46,342 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:35:46,342 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35fa9ee617f001aec9f096271ca1ed8:C 2024-12-03T02:35:50,004 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T02:35:52,925 INFO [Thread-1944 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1944. 2024-12-03T02:35:52,925 DEBUG [Thread-1944 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:35:52,925 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:52,926 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:52,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 216 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 192 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 200 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 222 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1379 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4136 rows 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1363 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4087 rows 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1384 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4151 rows 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1382 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4141 rows 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1366 2024-12-03T02:35:52,926 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4096 rows 2024-12-03T02:35:52,926 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:35:52,926 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:35:52,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:52,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:52,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:35:52,927 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T02:35:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T02:35:52,929 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193352929"}]},"ts":"1733193352929"} 2024-12-03T02:35:52,930 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T02:35:52,930 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T02:35:52,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T02:35:52,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35fa9ee617f001aec9f096271ca1ed8, UNASSIGN}] 2024-12-03T02:35:52,933 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35fa9ee617f001aec9f096271ca1ed8, UNASSIGN 2024-12-03T02:35:52,933 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=b35fa9ee617f001aec9f096271ca1ed8, regionState=CLOSING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:52,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35fa9ee617f001aec9f096271ca1ed8, UNASSIGN because future has completed 2024-12-03T02:35:52,935 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:35:52,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:35:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T02:35:53,087 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:53,087 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:35:53,087 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing b35fa9ee617f001aec9f096271ca1ed8, disabling compactions & flushes 2024-12-03T02:35:53,087 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:53,087 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:53,087 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. after waiting 0 ms 2024-12-03T02:35:53,087 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:53,088 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing b35fa9ee617f001aec9f096271ca1ed8 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-03T02:35:53,088 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=A 2024-12-03T02:35:53,088 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:53,088 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=B 2024-12-03T02:35:53,088 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:53,088 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b35fa9ee617f001aec9f096271ca1ed8, store=C 2024-12-03T02:35:53,088 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:53,092 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/6d4cb707c4b642d28b5792b5604b5d81 is 50, key is test_row_1/A:col10/1733193352924/Put/seqid=0 2024-12-03T02:35:53,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742359_1535 (size=9857) 2024-12-03T02:35:53,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T02:35:53,497 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1011 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/6d4cb707c4b642d28b5792b5604b5d81 2024-12-03T02:35:53,506 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/0da4c5717c4d47eca36dc454224dac09 is 50, key is test_row_1/B:col10/1733193352924/Put/seqid=0 2024-12-03T02:35:53,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742360_1536 (size=9857) 2024-12-03T02:35:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T02:35:53,924 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1011 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/0da4c5717c4d47eca36dc454224dac09 2024-12-03T02:35:53,929 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/079f7817669c4965931d51e195559ced is 50, key is test_row_1/C:col10/1733193352924/Put/seqid=0 2024-12-03T02:35:53,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742361_1537 (size=9857) 2024-12-03T02:35:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T02:35:54,333 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=1011 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/079f7817669c4965931d51e195559ced 2024-12-03T02:35:54,338 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/A/6d4cb707c4b642d28b5792b5604b5d81 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/6d4cb707c4b642d28b5792b5604b5d81 2024-12-03T02:35:54,343 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/6d4cb707c4b642d28b5792b5604b5d81, entries=100, sequenceid=1011, filesize=9.6 K 2024-12-03T02:35:54,344 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/B/0da4c5717c4d47eca36dc454224dac09 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0da4c5717c4d47eca36dc454224dac09 2024-12-03T02:35:54,348 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0da4c5717c4d47eca36dc454224dac09, entries=100, sequenceid=1011, filesize=9.6 K 2024-12-03T02:35:54,349 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/.tmp/C/079f7817669c4965931d51e195559ced as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/079f7817669c4965931d51e195559ced 2024-12-03T02:35:54,353 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/079f7817669c4965931d51e195559ced, entries=100, sequenceid=1011, filesize=9.6 K 2024-12-03T02:35:54,354 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b35fa9ee617f001aec9f096271ca1ed8 in 1266ms, sequenceid=1011, compaction requested=false 2024-12-03T02:35:54,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/26bbc85863b8488c8154c53aa042ed0b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/85fd94f2bc114711a6349c1f3262ca81, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5afb394ca0b240818bcb94d0b282df25, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/bb2a22968d0d46b5b47ebb4c9e3d028a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/c507ae6169bf4f29906571cf783c1123, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/4574aceb5f534eb0941d3396f633172e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e84cd211b5df43769dc8f7e2b2ab4a7a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/70ecaf20c4db431ebc427bcfa4f5ad88, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f60af39a933341a4acbe5a2899e606c7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/63a5e1633665445cb49d2c783c322269, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f3ce05a5fa6f4ab2aeb95bfc69c7c5ff, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b059257bb9d94736ae511eb98fefb67e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/6866dd8e8de2423f90f39f97c5366253, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e45ad668f3674da2b5182bd86c521e17, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/69ca27cc0e9d4d99a972ef54c4ee4b98, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/19ec8550e9684213ad7cc45d918c2691, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0bc3c685a4d44e379bf23caae26fc082, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/cd80d3c8a5414e07a7c8fed489b8d232, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b08a46ab5c464533910d91c7e09ab33f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/28244b971e5b47ed9729d21d320b9e59, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dd625a5a82c44df68c289cc850f767fb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/35bf35e763544f959d158a41936cc349, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e7cbbd74a90d4b41ac54e1db41197a88, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b3bb2a37a37940f68384bcf88796e92a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/7016c8f666b6442a908f8db36670ddab, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0f557b62d265404ea6653dd8ea72ac8a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/561227140e204bb99ab493456a502c7d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd5cebb5b1ad4ca5985b6a9eb585654e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd167f8948484e6e841e9e28448a71f4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5ff1a81e81fd49659b71bedd16227912, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dfdff82d0dfc4c2396c466a98bea9908, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/99e64d0794c84cc18298c6f6d7e9c341, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/d26e3baaad9d4b868672e913a48e3bfa] to archive 2024-12-03T02:35:54,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:35:54,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/26bbc85863b8488c8154c53aa042ed0b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/26bbc85863b8488c8154c53aa042ed0b 2024-12-03T02:35:54,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/85fd94f2bc114711a6349c1f3262ca81 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/85fd94f2bc114711a6349c1f3262ca81 2024-12-03T02:35:54,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5afb394ca0b240818bcb94d0b282df25 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5afb394ca0b240818bcb94d0b282df25 2024-12-03T02:35:54,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/bb2a22968d0d46b5b47ebb4c9e3d028a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/bb2a22968d0d46b5b47ebb4c9e3d028a 2024-12-03T02:35:54,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/c507ae6169bf4f29906571cf783c1123 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/c507ae6169bf4f29906571cf783c1123 2024-12-03T02:35:54,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/4574aceb5f534eb0941d3396f633172e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/4574aceb5f534eb0941d3396f633172e 2024-12-03T02:35:54,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e84cd211b5df43769dc8f7e2b2ab4a7a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e84cd211b5df43769dc8f7e2b2ab4a7a 2024-12-03T02:35:54,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/70ecaf20c4db431ebc427bcfa4f5ad88 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/70ecaf20c4db431ebc427bcfa4f5ad88 2024-12-03T02:35:54,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f60af39a933341a4acbe5a2899e606c7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f60af39a933341a4acbe5a2899e606c7 2024-12-03T02:35:54,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/63a5e1633665445cb49d2c783c322269 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/63a5e1633665445cb49d2c783c322269 2024-12-03T02:35:54,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f3ce05a5fa6f4ab2aeb95bfc69c7c5ff to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/f3ce05a5fa6f4ab2aeb95bfc69c7c5ff 2024-12-03T02:35:54,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b059257bb9d94736ae511eb98fefb67e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b059257bb9d94736ae511eb98fefb67e 2024-12-03T02:35:54,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/6866dd8e8de2423f90f39f97c5366253 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/6866dd8e8de2423f90f39f97c5366253 2024-12-03T02:35:54,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e45ad668f3674da2b5182bd86c521e17 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e45ad668f3674da2b5182bd86c521e17 2024-12-03T02:35:54,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/69ca27cc0e9d4d99a972ef54c4ee4b98 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/69ca27cc0e9d4d99a972ef54c4ee4b98 2024-12-03T02:35:54,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/19ec8550e9684213ad7cc45d918c2691 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/19ec8550e9684213ad7cc45d918c2691 2024-12-03T02:35:54,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0bc3c685a4d44e379bf23caae26fc082 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0bc3c685a4d44e379bf23caae26fc082 2024-12-03T02:35:54,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/cd80d3c8a5414e07a7c8fed489b8d232 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/cd80d3c8a5414e07a7c8fed489b8d232 2024-12-03T02:35:54,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b08a46ab5c464533910d91c7e09ab33f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b08a46ab5c464533910d91c7e09ab33f 2024-12-03T02:35:54,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/28244b971e5b47ed9729d21d320b9e59 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/28244b971e5b47ed9729d21d320b9e59 2024-12-03T02:35:54,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dd625a5a82c44df68c289cc850f767fb to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dd625a5a82c44df68c289cc850f767fb 2024-12-03T02:35:54,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/35bf35e763544f959d158a41936cc349 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/35bf35e763544f959d158a41936cc349 2024-12-03T02:35:54,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e7cbbd74a90d4b41ac54e1db41197a88 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/e7cbbd74a90d4b41ac54e1db41197a88 2024-12-03T02:35:54,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b3bb2a37a37940f68384bcf88796e92a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/b3bb2a37a37940f68384bcf88796e92a 2024-12-03T02:35:54,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/7016c8f666b6442a908f8db36670ddab to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/7016c8f666b6442a908f8db36670ddab 2024-12-03T02:35:54,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0f557b62d265404ea6653dd8ea72ac8a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/0f557b62d265404ea6653dd8ea72ac8a 2024-12-03T02:35:54,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/561227140e204bb99ab493456a502c7d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/561227140e204bb99ab493456a502c7d 2024-12-03T02:35:54,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd5cebb5b1ad4ca5985b6a9eb585654e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd5cebb5b1ad4ca5985b6a9eb585654e 2024-12-03T02:35:54,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd167f8948484e6e841e9e28448a71f4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/fd167f8948484e6e841e9e28448a71f4 2024-12-03T02:35:54,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5ff1a81e81fd49659b71bedd16227912 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/5ff1a81e81fd49659b71bedd16227912 2024-12-03T02:35:54,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dfdff82d0dfc4c2396c466a98bea9908 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/dfdff82d0dfc4c2396c466a98bea9908 2024-12-03T02:35:54,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/99e64d0794c84cc18298c6f6d7e9c341 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/99e64d0794c84cc18298c6f6d7e9c341 2024-12-03T02:35:54,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/d26e3baaad9d4b868672e913a48e3bfa to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/d26e3baaad9d4b868672e913a48e3bfa 2024-12-03T02:35:54,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/4e1ff90e78cb41d8b1e661bd3dc95b60, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9506ca6a3f8a432796be85f03875b53d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fbea8cf8d4c44999bf8bbfc833a2258d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8f7ab715731749788b2b8a4161a19b10, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/02b4daf7f0ed4e73a1e210e14002ad38, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/3bf1441aa00b4374a3f08f4f3b60bf78, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b794e1cf0b1149f3b7828e88d0699518, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9cb79802939d451e978dc37bdb618a1c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8dbb7662a85d45468f29be3128c650bb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/af6608b0a2ce4ecbaea07b8dda758395, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9fb554d6aec3466cb6ebc6aae8aba428, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/d5d919c67b7442a4981513e95ba46108, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/bff6806072a7477f805c0062900941f1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/2da7fd869b2b4c8cb66f1cc0998d2e74, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/162e4446ffd540e5bd27f38debd48eba, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b33fe2b47680404dbb32afbf4a63066b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fa5bf3a930d2434d9f269a6782cce850, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/67a2f2fc9ae84ad5907af988ebbfd7e6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/6b9d374ab6604a4db28296d94d22c1c0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d3d2d55357b4c6da5936d5a48e80200, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/742d698bf1d24855a4b2531e631838d4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b86d4fbddbdd407abea75a09e93646ac, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c62267068d9e433c8dfb434c042a743c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/aed9f9b7705d45d1b1d4245a6bbd038c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/436f30333d4f4bd4be29a339bf70ad55, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/ae4b892ca74a46589a0fd5ae9551f332, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0931183511f540dcb03a5fb3bf398730, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/237e2168f3b04a34869cdf551de97e37, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/e4531c9c9bef41e5ba0715c989e6e7a7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c4c36b425ee24ac38d86754c77d05304, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d0dc2a3282946be8055b8728c90457e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/5ddd1750bea24d1285d6b6b8f4a8e9cb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/76d244dd2a3341f3bbdc70d326419c9b] to archive 2024-12-03T02:35:54,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:35:54,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/4e1ff90e78cb41d8b1e661bd3dc95b60 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/4e1ff90e78cb41d8b1e661bd3dc95b60 2024-12-03T02:35:54,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9506ca6a3f8a432796be85f03875b53d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9506ca6a3f8a432796be85f03875b53d 2024-12-03T02:35:54,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fbea8cf8d4c44999bf8bbfc833a2258d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fbea8cf8d4c44999bf8bbfc833a2258d 2024-12-03T02:35:54,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8f7ab715731749788b2b8a4161a19b10 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8f7ab715731749788b2b8a4161a19b10 2024-12-03T02:35:54,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/02b4daf7f0ed4e73a1e210e14002ad38 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/02b4daf7f0ed4e73a1e210e14002ad38 2024-12-03T02:35:54,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/3bf1441aa00b4374a3f08f4f3b60bf78 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/3bf1441aa00b4374a3f08f4f3b60bf78 2024-12-03T02:35:54,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b794e1cf0b1149f3b7828e88d0699518 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b794e1cf0b1149f3b7828e88d0699518 2024-12-03T02:35:54,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9cb79802939d451e978dc37bdb618a1c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9cb79802939d451e978dc37bdb618a1c 2024-12-03T02:35:54,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8dbb7662a85d45468f29be3128c650bb to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/8dbb7662a85d45468f29be3128c650bb 2024-12-03T02:35:54,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/af6608b0a2ce4ecbaea07b8dda758395 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/af6608b0a2ce4ecbaea07b8dda758395 2024-12-03T02:35:54,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9fb554d6aec3466cb6ebc6aae8aba428 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/9fb554d6aec3466cb6ebc6aae8aba428 2024-12-03T02:35:54,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/d5d919c67b7442a4981513e95ba46108 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/d5d919c67b7442a4981513e95ba46108 2024-12-03T02:35:54,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/bff6806072a7477f805c0062900941f1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/bff6806072a7477f805c0062900941f1 2024-12-03T02:35:54,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/2da7fd869b2b4c8cb66f1cc0998d2e74 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/2da7fd869b2b4c8cb66f1cc0998d2e74 2024-12-03T02:35:54,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/162e4446ffd540e5bd27f38debd48eba to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/162e4446ffd540e5bd27f38debd48eba 2024-12-03T02:35:54,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b33fe2b47680404dbb32afbf4a63066b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b33fe2b47680404dbb32afbf4a63066b 2024-12-03T02:35:54,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fa5bf3a930d2434d9f269a6782cce850 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/fa5bf3a930d2434d9f269a6782cce850 2024-12-03T02:35:54,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/67a2f2fc9ae84ad5907af988ebbfd7e6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/67a2f2fc9ae84ad5907af988ebbfd7e6 2024-12-03T02:35:54,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/6b9d374ab6604a4db28296d94d22c1c0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/6b9d374ab6604a4db28296d94d22c1c0 2024-12-03T02:35:54,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d3d2d55357b4c6da5936d5a48e80200 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d3d2d55357b4c6da5936d5a48e80200 2024-12-03T02:35:54,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/742d698bf1d24855a4b2531e631838d4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/742d698bf1d24855a4b2531e631838d4 2024-12-03T02:35:54,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b86d4fbddbdd407abea75a09e93646ac to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/b86d4fbddbdd407abea75a09e93646ac 2024-12-03T02:35:54,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c62267068d9e433c8dfb434c042a743c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c62267068d9e433c8dfb434c042a743c 2024-12-03T02:35:54,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/aed9f9b7705d45d1b1d4245a6bbd038c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/aed9f9b7705d45d1b1d4245a6bbd038c 2024-12-03T02:35:54,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/436f30333d4f4bd4be29a339bf70ad55 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/436f30333d4f4bd4be29a339bf70ad55 2024-12-03T02:35:54,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/ae4b892ca74a46589a0fd5ae9551f332 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/ae4b892ca74a46589a0fd5ae9551f332 2024-12-03T02:35:54,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0931183511f540dcb03a5fb3bf398730 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0931183511f540dcb03a5fb3bf398730 2024-12-03T02:35:54,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/237e2168f3b04a34869cdf551de97e37 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/237e2168f3b04a34869cdf551de97e37 2024-12-03T02:35:54,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/e4531c9c9bef41e5ba0715c989e6e7a7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/e4531c9c9bef41e5ba0715c989e6e7a7 2024-12-03T02:35:54,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c4c36b425ee24ac38d86754c77d05304 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/c4c36b425ee24ac38d86754c77d05304 2024-12-03T02:35:54,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d0dc2a3282946be8055b8728c90457e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/7d0dc2a3282946be8055b8728c90457e 2024-12-03T02:35:54,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/5ddd1750bea24d1285d6b6b8f4a8e9cb to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/5ddd1750bea24d1285d6b6b8f4a8e9cb 2024-12-03T02:35:54,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/76d244dd2a3341f3bbdc70d326419c9b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/76d244dd2a3341f3bbdc70d326419c9b 2024-12-03T02:35:54,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/11011d1a3dc048339c916872ad515b3e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/66e321f148834ee0b5b745d07505875c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/632fb5640a714cb98ccdd0d749195c92, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7a69a056ae2942939c53e18ecce92d0d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b2a32b5ecd4846cfac4c02b1b412aca5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b3eeac80600b49d3b7aec41ccf633c8a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c2a2a4e2b034723b73b4a73412a95df, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8cdee0a2f6354a56a1347d91a7e9a45a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/fd7db597c0e44382b3420e682fc8d217, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4379ecbe47124eeeb698bf17831a4200, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/73a68c2fcbf34383903e4dafd67a35ce, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/5973b977d58046b393f838adf2c41e77, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/54a4da55950a4b14853a01f201fb95b8, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7dbed7ae1d8642eb842f104cbe670126, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/2a57811a31a64aba95289adb124f5633, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/ce6477c7840b43b7a46621298d26e671, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f269b072ab447478a25f647e769407a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b71c6114771b444dad9ebc0c7b4fb73c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f4908225d234803a85b9bf510c27144, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/446c25a5cad4482487b2ce279090c486, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1f3fdc0af95f431e847f2fb45b49be57, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c7526db8c5641399741571fee90b0cb, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/a6f76c32d15f43b8b0d44b447f0ceaa7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8d5b822b6a8b420a9ab7eddfdb47a232, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/cfbd1622bfa54ee9b92ab7b633b6185e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/0fc22cbac0a249e1ab8faf2069c7c143, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8a75aece25534afdb444bf67f2b820a4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/223e231d355b45d680c8b63bb66797b4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/c9c602157cb143169a4bd4bbb9a49bb9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/6c2cf25818e14e76adbf11db1a8a492c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/e597d2736cc74423863b75751a0e49ce, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1a700fd0ccdf46c6883bc4ba4e7b8fad, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/00b500de60c340648be0aedad6269be3] to archive 2024-12-03T02:35:54,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:35:54,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/11011d1a3dc048339c916872ad515b3e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/11011d1a3dc048339c916872ad515b3e 2024-12-03T02:35:54,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/66e321f148834ee0b5b745d07505875c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/66e321f148834ee0b5b745d07505875c 2024-12-03T02:35:54,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/632fb5640a714cb98ccdd0d749195c92 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/632fb5640a714cb98ccdd0d749195c92 2024-12-03T02:35:54,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7a69a056ae2942939c53e18ecce92d0d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7a69a056ae2942939c53e18ecce92d0d 2024-12-03T02:35:54,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b2a32b5ecd4846cfac4c02b1b412aca5 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b2a32b5ecd4846cfac4c02b1b412aca5 2024-12-03T02:35:54,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b3eeac80600b49d3b7aec41ccf633c8a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b3eeac80600b49d3b7aec41ccf633c8a 2024-12-03T02:35:54,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c2a2a4e2b034723b73b4a73412a95df to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c2a2a4e2b034723b73b4a73412a95df 2024-12-03T02:35:54,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8cdee0a2f6354a56a1347d91a7e9a45a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8cdee0a2f6354a56a1347d91a7e9a45a 2024-12-03T02:35:54,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/fd7db597c0e44382b3420e682fc8d217 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/fd7db597c0e44382b3420e682fc8d217 2024-12-03T02:35:54,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4379ecbe47124eeeb698bf17831a4200 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4379ecbe47124eeeb698bf17831a4200 2024-12-03T02:35:54,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/73a68c2fcbf34383903e4dafd67a35ce to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/73a68c2fcbf34383903e4dafd67a35ce 2024-12-03T02:35:54,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/5973b977d58046b393f838adf2c41e77 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/5973b977d58046b393f838adf2c41e77 2024-12-03T02:35:54,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/54a4da55950a4b14853a01f201fb95b8 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/54a4da55950a4b14853a01f201fb95b8 2024-12-03T02:35:54,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7dbed7ae1d8642eb842f104cbe670126 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/7dbed7ae1d8642eb842f104cbe670126 2024-12-03T02:35:54,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/2a57811a31a64aba95289adb124f5633 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/2a57811a31a64aba95289adb124f5633 2024-12-03T02:35:54,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/ce6477c7840b43b7a46621298d26e671 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/ce6477c7840b43b7a46621298d26e671 2024-12-03T02:35:54,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f269b072ab447478a25f647e769407a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f269b072ab447478a25f647e769407a 2024-12-03T02:35:54,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b71c6114771b444dad9ebc0c7b4fb73c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/b71c6114771b444dad9ebc0c7b4fb73c 2024-12-03T02:35:54,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f4908225d234803a85b9bf510c27144 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/9f4908225d234803a85b9bf510c27144 2024-12-03T02:35:54,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/446c25a5cad4482487b2ce279090c486 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/446c25a5cad4482487b2ce279090c486 2024-12-03T02:35:54,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1f3fdc0af95f431e847f2fb45b49be57 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1f3fdc0af95f431e847f2fb45b49be57 2024-12-03T02:35:54,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c7526db8c5641399741571fee90b0cb to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/4c7526db8c5641399741571fee90b0cb 2024-12-03T02:35:54,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/a6f76c32d15f43b8b0d44b447f0ceaa7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/a6f76c32d15f43b8b0d44b447f0ceaa7 2024-12-03T02:35:54,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8d5b822b6a8b420a9ab7eddfdb47a232 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8d5b822b6a8b420a9ab7eddfdb47a232 2024-12-03T02:35:54,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/cfbd1622bfa54ee9b92ab7b633b6185e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/cfbd1622bfa54ee9b92ab7b633b6185e 2024-12-03T02:35:54,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/0fc22cbac0a249e1ab8faf2069c7c143 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/0fc22cbac0a249e1ab8faf2069c7c143 2024-12-03T02:35:54,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8a75aece25534afdb444bf67f2b820a4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/8a75aece25534afdb444bf67f2b820a4 2024-12-03T02:35:54,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/223e231d355b45d680c8b63bb66797b4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/223e231d355b45d680c8b63bb66797b4 2024-12-03T02:35:54,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/c9c602157cb143169a4bd4bbb9a49bb9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/c9c602157cb143169a4bd4bbb9a49bb9 2024-12-03T02:35:54,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/6c2cf25818e14e76adbf11db1a8a492c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/6c2cf25818e14e76adbf11db1a8a492c 2024-12-03T02:35:54,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/e597d2736cc74423863b75751a0e49ce to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/e597d2736cc74423863b75751a0e49ce 2024-12-03T02:35:54,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1a700fd0ccdf46c6883bc4ba4e7b8fad to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/1a700fd0ccdf46c6883bc4ba4e7b8fad 2024-12-03T02:35:54,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/00b500de60c340648be0aedad6269be3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/00b500de60c340648be0aedad6269be3 2024-12-03T02:35:54,540 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/recovered.edits/1014.seqid, newMaxSeqId=1014, maxSeqId=1 2024-12-03T02:35:54,540 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. 2024-12-03T02:35:54,541 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for b35fa9ee617f001aec9f096271ca1ed8: Waiting for close lock at 1733193353087Running coprocessor pre-close hooks at 1733193353087Disabling compacts and flushes for region at 1733193353087Disabling writes for close at 1733193353087Obtaining lock to block concurrent updates at 1733193353088 (+1 ms)Preparing flush snapshotting stores in b35fa9ee617f001aec9f096271ca1ed8 at 1733193353088Finished memstore snapshotting TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733193353088Flushing stores of TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8. at 1733193353089 (+1 ms)Flushing b35fa9ee617f001aec9f096271ca1ed8/A: creating writer at 1733193353089Flushing b35fa9ee617f001aec9f096271ca1ed8/A: appending metadata at 1733193353091 (+2 ms)Flushing b35fa9ee617f001aec9f096271ca1ed8/A: closing flushed file at 1733193353091Flushing b35fa9ee617f001aec9f096271ca1ed8/B: creating writer at 1733193353502 (+411 ms)Flushing b35fa9ee617f001aec9f096271ca1ed8/B: appending metadata at 1733193353506 (+4 ms)Flushing b35fa9ee617f001aec9f096271ca1ed8/B: closing flushed file at 1733193353506Flushing b35fa9ee617f001aec9f096271ca1ed8/C: creating writer at 1733193353927 (+421 ms)Flushing b35fa9ee617f001aec9f096271ca1ed8/C: appending metadata at 1733193353929 (+2 ms)Flushing b35fa9ee617f001aec9f096271ca1ed8/C: closing flushed file at 1733193353929Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@787e2fc9: reopening flushed file at 1733193354337 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55fee723: reopening flushed file at 1733193354343 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68a6d193: reopening flushed file at 1733193354348 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for b35fa9ee617f001aec9f096271ca1ed8 in 1266ms, sequenceid=1011, compaction requested=false at 1733193354354 (+6 ms)Writing region close event to WAL at 1733193354531 (+177 ms)Running coprocessor post-close hooks at 1733193354540 (+9 ms)Closed at 1733193354540 2024-12-03T02:35:54,543 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:54,543 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=b35fa9ee617f001aec9f096271ca1ed8, regionState=CLOSED 2024-12-03T02:35:54,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:35:54,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-03T02:35:54,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure b35fa9ee617f001aec9f096271ca1ed8, server=64035cbb0b46,38721,1733193202350 in 1.6100 sec 2024-12-03T02:35:54,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-03T02:35:54,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35fa9ee617f001aec9f096271ca1ed8, UNASSIGN in 1.6150 sec 2024-12-03T02:35:54,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-03T02:35:54,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6190 sec 2024-12-03T02:35:54,553 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193354553"}]},"ts":"1733193354553"} 2024-12-03T02:35:54,555 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T02:35:54,555 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T02:35:54,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6290 sec 2024-12-03T02:35:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T02:35:55,075 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:55,076 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T02:35:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:55,078 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T02:35:55,079 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:55,081 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:55,088 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/recovered.edits] 2024-12-03T02:35:55,093 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/6d4cb707c4b642d28b5792b5604b5d81 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/6d4cb707c4b642d28b5792b5604b5d81 2024-12-03T02:35:55,094 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/a47e5ce5a6394c4b8113a19fb26cd33b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/A/a47e5ce5a6394c4b8113a19fb26cd33b 2024-12-03T02:35:55,097 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0da4c5717c4d47eca36dc454224dac09 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/0da4c5717c4d47eca36dc454224dac09 2024-12-03T02:35:55,098 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/426740060f8744b4ab4cea4354d2457e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/B/426740060f8744b4ab4cea4354d2457e 2024-12-03T02:35:55,100 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/079f7817669c4965931d51e195559ced to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/079f7817669c4965931d51e195559ced 2024-12-03T02:35:55,102 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/870ffcbe35cf47a6bae2f63e5ce68083 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/C/870ffcbe35cf47a6bae2f63e5ce68083 2024-12-03T02:35:55,104 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/recovered.edits/1014.seqid to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8/recovered.edits/1014.seqid 2024-12-03T02:35:55,105 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/b35fa9ee617f001aec9f096271ca1ed8 2024-12-03T02:35:55,105 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T02:35:55,108 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:55,110 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T02:35:55,112 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T02:35:55,113 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:55,113 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T02:35:55,114 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733193355113"}]},"ts":"9223372036854775807"} 2024-12-03T02:35:55,116 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T02:35:55,116 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b35fa9ee617f001aec9f096271ca1ed8, NAME => 'TestAcidGuarantees,,1733193321741.b35fa9ee617f001aec9f096271ca1ed8.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T02:35:55,116 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T02:35:55,116 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733193355116"}]},"ts":"9223372036854775807"} 2024-12-03T02:35:55,118 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T02:35:55,119 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:55,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-12-03T02:35:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T02:35:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T02:35:55,188 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:55,194 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 251), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=664 (was 787), ProcessCount=9 (was 9), AvailableMemoryMB=3729 (was 2732) - AvailableMemoryMB LEAK? - 2024-12-03T02:35:55,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=664, ProcessCount=9, AvailableMemoryMB=3728 2024-12-03T02:35:55,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T02:35:55,204 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:35:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:55,205 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T02:35:55,206 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:55,206 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-03T02:35:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T02:35:55,207 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T02:35:55,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742362_1538 (size=1061) 2024-12-03T02:35:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T02:35:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T02:35:55,614 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37 2024-12-03T02:35:55,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742363_1539 (size=53) 2024-12-03T02:35:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T02:35:56,030 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:35:56,030 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0be0304c4b25813eaf7af443d2898000, disabling compactions & flushes 2024-12-03T02:35:56,030 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:56,030 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:56,030 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. after waiting 0 ms 2024-12-03T02:35:56,030 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:56,030 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:56,030 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0be0304c4b25813eaf7af443d2898000: Waiting for close lock at 1733193356030Disabling compacts and flushes for region at 1733193356030Disabling writes for close at 1733193356030Writing region close event to WAL at 1733193356030Closed at 1733193356030 2024-12-03T02:35:56,032 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T02:35:56,032 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733193356032"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733193356032"}]},"ts":"1733193356032"} 2024-12-03T02:35:56,034 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T02:35:56,035 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T02:35:56,035 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193356035"}]},"ts":"1733193356035"} 2024-12-03T02:35:56,036 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T02:35:56,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, ASSIGN}] 2024-12-03T02:35:56,037 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, ASSIGN 2024-12-03T02:35:56,038 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, ASSIGN; state=OFFLINE, location=64035cbb0b46,38721,1733193202350; forceNewPlan=false, retain=false 2024-12-03T02:35:56,188 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0be0304c4b25813eaf7af443d2898000, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:56,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, ASSIGN because future has completed 2024-12-03T02:35:56,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:35:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T02:35:56,345 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:56,345 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:35:56,346 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,346 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:35:56,346 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,346 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,347 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,348 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:35:56,349 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be0304c4b25813eaf7af443d2898000 columnFamilyName A 2024-12-03T02:35:56,349 DEBUG [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:56,349 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(327): Store=0be0304c4b25813eaf7af443d2898000/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:35:56,349 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,351 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:35:56,351 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be0304c4b25813eaf7af443d2898000 columnFamilyName B 2024-12-03T02:35:56,351 DEBUG [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:56,351 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(327): Store=0be0304c4b25813eaf7af443d2898000/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:35:56,352 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,353 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:35:56,353 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be0304c4b25813eaf7af443d2898000 columnFamilyName C 2024-12-03T02:35:56,353 DEBUG [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:56,353 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(327): Store=0be0304c4b25813eaf7af443d2898000/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:35:56,354 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:56,354 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,355 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,355 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,356 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,356 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,356 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:35:56,358 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,371 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:35:56,374 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 0be0304c4b25813eaf7af443d2898000; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70225594, jitterRate=0.046442896127700806}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:35:56,374 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:56,375 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 0be0304c4b25813eaf7af443d2898000: Running coprocessor pre-open hook at 1733193356346Writing region info on filesystem at 1733193356346Initializing all the Stores at 1733193356347 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193356347Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193356347Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193356347Cleaning up temporary data from old regions at 1733193356356 (+9 ms)Running coprocessor post-open hooks at 1733193356375 (+19 ms)Region opened successfully at 1733193356375 2024-12-03T02:35:56,376 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., pid=144, masterSystemTime=1733193356342 2024-12-03T02:35:56,379 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:56,379 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:56,379 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=0be0304c4b25813eaf7af443d2898000, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:56,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:35:56,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-03T02:35:56,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 in 196 msec 2024-12-03T02:35:56,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-03T02:35:56,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, ASSIGN in 367 msec 2024-12-03T02:35:56,407 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T02:35:56,407 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193356407"}]},"ts":"1733193356407"} 2024-12-03T02:35:56,409 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T02:35:56,410 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T02:35:56,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2060 sec 2024-12-03T02:35:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T02:35:57,355 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:57,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:57,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:57,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:57,357 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c8a0c84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:57,359 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb66b2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:57,360 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:57,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:57,362 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T02:35:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 64035cbb0b46,44739,1733193201066 2024-12-03T02:35:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1eca0055 2024-12-03T02:35:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T02:35:57,364 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T02:35:57,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T02:35:57,365 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:35:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T02:35:57,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742364_1540 (size=1097) 2024-12-03T02:35:57,775 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T02:35:57,775 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T02:35:57,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T02:35:57,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, REOPEN/MOVE}] 2024-12-03T02:35:57,781 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, REOPEN/MOVE 2024-12-03T02:35:57,782 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=0be0304c4b25813eaf7af443d2898000, regionState=CLOSING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:57,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, REOPEN/MOVE because future has completed 2024-12-03T02:35:57,783 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:35:57,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:35:57,936 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:57,936 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:35:57,936 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 0be0304c4b25813eaf7af443d2898000, disabling compactions & flushes 2024-12-03T02:35:57,936 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:57,936 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:57,936 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. after waiting 0 ms 2024-12-03T02:35:57,936 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:57,940 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T02:35:57,940 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:57,941 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 0be0304c4b25813eaf7af443d2898000: Waiting for close lock at 1733193357936Running coprocessor pre-close hooks at 1733193357936Disabling compacts and flushes for region at 1733193357936Disabling writes for close at 1733193357936Writing region close event to WAL at 1733193357937 (+1 ms)Running coprocessor post-close hooks at 1733193357940 (+3 ms)Closed at 1733193357940 2024-12-03T02:35:57,941 WARN [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 0be0304c4b25813eaf7af443d2898000 to self. 2024-12-03T02:35:57,942 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:57,943 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=0be0304c4b25813eaf7af443d2898000, regionState=CLOSED 2024-12-03T02:35:57,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:35:57,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-03T02:35:57,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 in 162 msec 2024-12-03T02:35:57,946 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, REOPEN/MOVE; state=CLOSED, location=64035cbb0b46,38721,1733193202350; forceNewPlan=false, retain=true 2024-12-03T02:35:58,097 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=0be0304c4b25813eaf7af443d2898000, regionState=OPENING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, REOPEN/MOVE because future has completed 2024-12-03T02:35:58,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:35:58,254 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,254 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:35:58,255 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,255 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:35:58,255 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,255 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,257 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,258 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:35:58,258 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be0304c4b25813eaf7af443d2898000 columnFamilyName A 2024-12-03T02:35:58,259 DEBUG [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:58,260 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(327): Store=0be0304c4b25813eaf7af443d2898000/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:35:58,261 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,261 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:35:58,261 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be0304c4b25813eaf7af443d2898000 columnFamilyName B 2024-12-03T02:35:58,261 DEBUG [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:58,262 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(327): Store=0be0304c4b25813eaf7af443d2898000/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:35:58,262 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,263 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T02:35:58,263 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0be0304c4b25813eaf7af443d2898000 columnFamilyName C 2024-12-03T02:35:58,263 DEBUG [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:58,263 INFO [StoreOpener-0be0304c4b25813eaf7af443d2898000-1 {}] regionserver.HStore(327): Store=0be0304c4b25813eaf7af443d2898000/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:35:58,264 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,264 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,264 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,266 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,267 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,267 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,267 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T02:35:58,269 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,269 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 0be0304c4b25813eaf7af443d2898000; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74463529, jitterRate=0.10959304869174957}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T02:35:58,269 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,270 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 0be0304c4b25813eaf7af443d2898000: Running coprocessor pre-open hook at 1733193358255Writing region info on filesystem at 1733193358255Initializing all the Stores at 1733193358256 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193358256Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193358257 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733193358257Cleaning up temporary data from old regions at 1733193358267 (+10 ms)Running coprocessor post-open hooks at 1733193358270 (+3 ms)Region opened successfully at 1733193358270 2024-12-03T02:35:58,271 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., pid=149, masterSystemTime=1733193358250 2024-12-03T02:35:58,274 DEBUG [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,274 INFO [RS_OPEN_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,274 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=0be0304c4b25813eaf7af443d2898000, regionState=OPEN, openSeqNum=5, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:35:58,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-03T02:35:58,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 in 178 msec 2024-12-03T02:35:58,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-03T02:35:58,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, REOPEN/MOVE in 499 msec 2024-12-03T02:35:58,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-03T02:35:58,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-12-03T02:35:58,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-12-03T02:35:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:35:58,293 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:35:58,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,301 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a372c2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65896b7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,312 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b8570f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,321 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e727c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,334 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2285e703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,343 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a0c5449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,352 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3179c3c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,356 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d79e76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,368 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d80a2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,372 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6078fe1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,385 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2c4667, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,388 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41111f29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,391 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41db6b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,392 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a4c89e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,396 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688b7549, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,397 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6019f20f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,401 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a4463e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,402 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e2fddb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,44739,-1 for getting cluster id 2024-12-03T02:35:58,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:35:58,406 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be4e62ab-a60c-4866-94e8-6b4f3144907d' 2024-12-03T02:35:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:35:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be4e62ab-a60c-4866-94e8-6b4f3144907d" 2024-12-03T02:35:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb828f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,44739,-1] 2024-12-03T02:35:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:35:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:35:58,412 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:35:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d193e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:35:58,413 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,414 DEBUG [Thread-2423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,414 DEBUG [Thread-2421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:35:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,415 DEBUG [Thread-2410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,416 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-03T02:35:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,417 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:35:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T02:35:58,418 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:35:58,418 DEBUG [Thread-2404 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,418 DEBUG [Thread-2412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:35:58,418 DEBUG [Thread-2404 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,419 DEBUG [Thread-2415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,420 DEBUG [Thread-2417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,420 DEBUG [Thread-2419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,420 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:35:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,38721,1733193202350, seqNum=-1] 2024-12-03T02:35:58,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:35:58,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:35:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5] 2024-12-03T02:35:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:35:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:35:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:35:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:35:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:35:58,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193418463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193418464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36054 deadline: 1733193418466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733193358450, max timestamp=1733193358467Num uniques 859; 2024-12-03T02:35:58,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:35:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193358450, max timestamp=1733193358467Num uniques 859; 2024-12-03T02:35:58,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:35:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193358450, max timestamp=1733193358467Num uniques 859; 2024-12-03T02:35:58,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:35:58,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120389fa18b8891f43c98ebf11a0606f3a85_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193358432/Put/seqid=0 2024-12-03T02:35:58,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742365_1541 (size=12154) 2024-12-03T02:35:58,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:58,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120389fa18b8891f43c98ebf11a0606f3a85_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389fa18b8891f43c98ebf11a0606f3a85_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:58,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/270d396adf234ab4a22aafc698d7674d, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:35:58,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/270d396adf234ab4a22aafc698d7674d is 175, key is test_row_0/A:col10/1733193358432/Put/seqid=0 2024-12-03T02:35:58,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742366_1542 (size=30955) 2024-12-03T02:35:58,509 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/270d396adf234ab4a22aafc698d7674d 2024-12-03T02:35:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T02:35:58,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/8dc719f62feb4f9fbff6a9d56e787ca6 is 50, key is test_row_0/B:col10/1733193358432/Put/seqid=0 2024-12-03T02:35:58,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742367_1543 (size=12001) 2024-12-03T02:35:58,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193418538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193418538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T02:35:58,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:35:58,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:58,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:58,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193418575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36054 deadline: 1733193418575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193418575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193418645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193418645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T02:35:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:35:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T02:35:58,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36054 deadline: 1733193418785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193418785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193418787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193418855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193418855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:58,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:58,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T02:35:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:35:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:58,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:58,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/8dc719f62feb4f9fbff6a9d56e787ca6 2024-12-03T02:35:59,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/64f1f7a3cb094cf0910550c8e188572d is 50, key is test_row_0/C:col10/1733193358432/Put/seqid=0 2024-12-03T02:35:59,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742368_1544 (size=12001) 2024-12-03T02:35:59,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/64f1f7a3cb094cf0910550c8e188572d 2024-12-03T02:35:59,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T02:35:59,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:59,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:35:59,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:59,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:59,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:35:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T02:35:59,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/270d396adf234ab4a22aafc698d7674d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/270d396adf234ab4a22aafc698d7674d 2024-12-03T02:35:59,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/270d396adf234ab4a22aafc698d7674d, entries=150, sequenceid=15, filesize=30.2 K 2024-12-03T02:35:59,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/8dc719f62feb4f9fbff6a9d56e787ca6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8dc719f62feb4f9fbff6a9d56e787ca6 2024-12-03T02:35:59,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193419095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193419095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36054 deadline: 1733193419095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8dc719f62feb4f9fbff6a9d56e787ca6, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T02:35:59,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/64f1f7a3cb094cf0910550c8e188572d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/64f1f7a3cb094cf0910550c8e188572d 2024-12-03T02:35:59,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/64f1f7a3cb094cf0910550c8e188572d, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T02:35:59,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=135.63 KB/138882 for 0be0304c4b25813eaf7af443d2898000 in 671ms, sequenceid=15, compaction requested=false 2024-12-03T02:35:59,108 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T02:35:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:35:59,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T02:35:59,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:59,188 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=135.63 KB heapSize=388.23 KB 2024-12-03T02:35:59,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193358497, max timestamp=1733193359167Num uniques 1009; 2024-12-03T02:35:59,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-03T02:35:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193358497, max timestamp=1733193359167Num uniques 1068; 2024-12-03T02:35:59,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T02:35:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:59,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:35:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:35:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:35:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:35:59,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:35:59,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120393f67103b82846d98dcda6ec493a0286_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193358516/Put/seqid=0 2024-12-03T02:35:59,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742369_1545 (size=12154) 2024-12-03T02:35:59,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:59,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193419255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193419258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,264 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120393f67103b82846d98dcda6ec493a0286_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120393f67103b82846d98dcda6ec493a0286_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:59,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/54a64c5cbabc4a6197216acbde78a534, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:35:59,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/54a64c5cbabc4a6197216acbde78a534 is 175, key is test_row_0/A:col10/1733193358516/Put/seqid=0 2024-12-03T02:35:59,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742370_1546 (size=30955) 2024-12-03T02:35:59,309 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/54a64c5cbabc4a6197216acbde78a534 2024-12-03T02:35:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/94cd0da605164fdca46e09f912e382b2 is 50, key is test_row_0/B:col10/1733193358516/Put/seqid=0 2024-12-03T02:35:59,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193419364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193419367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742371_1547 (size=12001) 2024-12-03T02:35:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T02:35:59,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193419578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193419578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193419605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193419605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36054 deadline: 1733193419605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,812 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/94cd0da605164fdca46e09f912e382b2 2024-12-03T02:35:59,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/29b56d7aa4114ba699294a7beeb3101a is 50, key is test_row_0/C:col10/1733193358516/Put/seqid=0 2024-12-03T02:35:59,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742372_1548 (size=12001) 2024-12-03T02:35:59,854 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48.80 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/29b56d7aa4114ba699294a7beeb3101a 2024-12-03T02:35:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/54a64c5cbabc4a6197216acbde78a534 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/54a64c5cbabc4a6197216acbde78a534 2024-12-03T02:35:59,865 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/54a64c5cbabc4a6197216acbde78a534, entries=150, sequenceid=54, filesize=30.2 K 2024-12-03T02:35:59,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/94cd0da605164fdca46e09f912e382b2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/94cd0da605164fdca46e09f912e382b2 2024-12-03T02:35:59,871 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/94cd0da605164fdca46e09f912e382b2, entries=150, sequenceid=54, filesize=11.7 K 2024-12-03T02:35:59,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/29b56d7aa4114ba699294a7beeb3101a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/29b56d7aa4114ba699294a7beeb3101a 2024-12-03T02:35:59,876 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/29b56d7aa4114ba699294a7beeb3101a, entries=150, sequenceid=54, filesize=11.7 K 2024-12-03T02:35:59,877 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~71.17 KB/72876, heapSize ~197.76 KB/202504, currentSize=107.34 KB/109920 for 0be0304c4b25813eaf7af443d2898000 in 689ms, sequenceid=54, compaction requested=false 2024-12-03T02:35:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:35:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:35:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-03T02:35:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-03T02:35:59,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-03T02:35:59,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-12-03T02:35:59,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.4650 sec 2024-12-03T02:35:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:59,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T02:35:59,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193359233, max timestamp=1733193359888Num uniques 859; 2024-12-03T02:35:59,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733193359233, max timestamp=1733193359888Num uniques 859; 2024-12-03T02:35:59,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:35:59,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:35:59,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:35:59,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:59,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:35:59,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:59,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:35:59,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:35:59,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036a04c7f8a02e4a16b6680e8d2819cbef_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193359888/Put/seqid=0 2024-12-03T02:35:59,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742373_1549 (size=14594) 2024-12-03T02:35:59,928 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:35:59,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036a04c7f8a02e4a16b6680e8d2819cbef_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036a04c7f8a02e4a16b6680e8d2819cbef_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:35:59,944 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/00d95a638efd4c108bddba062b1ec740, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:35:59,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/00d95a638efd4c108bddba062b1ec740 is 175, key is test_row_0/A:col10/1733193359888/Put/seqid=0 2024-12-03T02:35:59,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:35:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193419947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193419947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:35:59,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193359899, max timestamp=1733193359945Num uniques 859; 2024-12-03T02:35:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:35:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:35:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:35:59,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193359899, max timestamp=1733193359945Num uniques 859; 2024-12-03T02:35:59,955 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:35:59,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733193359899, max timestamp=1733193359945Num uniques 859; 2024-12-03T02:35:59,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T02:35:59,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742374_1550 (size=39549) 2024-12-03T02:35:59,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=75, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/00d95a638efd4c108bddba062b1ec740 2024-12-03T02:35:59,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/c808195946fe41f78a90e08db8ad8b0c is 50, key is test_row_0/B:col10/1733193359888/Put/seqid=0 2024-12-03T02:36:00,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742375_1551 (size=12001) 2024-12-03T02:36:00,057 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:36:00,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193420144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193420145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193420255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193420255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/c808195946fe41f78a90e08db8ad8b0c 2024-12-03T02:36:00,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/a0390e8b559a4dbd89fcb620a2034b47 is 50, key is test_row_0/C:col10/1733193359888/Put/seqid=0 2024-12-03T02:36:00,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193420465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193420466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742376_1552 (size=12001) 2024-12-03T02:36:00,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/a0390e8b559a4dbd89fcb620a2034b47 2024-12-03T02:36:00,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/00d95a638efd4c108bddba062b1ec740 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/00d95a638efd4c108bddba062b1ec740 2024-12-03T02:36:00,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/00d95a638efd4c108bddba062b1ec740, entries=200, sequenceid=75, filesize=38.6 K 2024-12-03T02:36:00,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/c808195946fe41f78a90e08db8ad8b0c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c808195946fe41f78a90e08db8ad8b0c 2024-12-03T02:36:00,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c808195946fe41f78a90e08db8ad8b0c, entries=150, sequenceid=75, filesize=11.7 K 2024-12-03T02:36:00,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/a0390e8b559a4dbd89fcb620a2034b47 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/a0390e8b559a4dbd89fcb620a2034b47 2024-12-03T02:36:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T02:36:00,571 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:00,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:36:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-03T02:36:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T02:36:00,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:36:00,590 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:36:00,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:36:00,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193420615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36054 deadline: 1733193420615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,621 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/a0390e8b559a4dbd89fcb620a2034b47, entries=150, sequenceid=75, filesize=11.7 K 2024-12-03T02:36:00,623 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:00,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~62.62 KB/64120, heapSize ~174.28 KB/178464, currentSize=126.29 KB/129318 for 0be0304c4b25813eaf7af443d2898000 in 737ms, sequenceid=75, compaction requested=true 2024-12-03T02:36:00,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:00,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:36:00,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:00,628 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:00,628 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:00,633 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:00,633 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/B is initiating minor compaction (all files) 2024-12-03T02:36:00,633 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/B in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:00,633 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8dc719f62feb4f9fbff6a9d56e787ca6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/94cd0da605164fdca46e09f912e382b2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c808195946fe41f78a90e08db8ad8b0c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=35.2 K 2024-12-03T02:36:00,633 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:36:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:00,633 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/A is initiating minor compaction (all files) 2024-12-03T02:36:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:36:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:00,633 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/A in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:00,634 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/270d396adf234ab4a22aafc698d7674d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/54a64c5cbabc4a6197216acbde78a534, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/00d95a638efd4c108bddba062b1ec740] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=99.1 K 2024-12-03T02:36:00,634 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc719f62feb4f9fbff6a9d56e787ca6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733193358431 2024-12-03T02:36:00,634 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:00,634 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/270d396adf234ab4a22aafc698d7674d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/54a64c5cbabc4a6197216acbde78a534, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/00d95a638efd4c108bddba062b1ec740] 2024-12-03T02:36:00,634 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 94cd0da605164fdca46e09f912e382b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733193358497 2024-12-03T02:36:00,634 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 270d396adf234ab4a22aafc698d7674d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733193358431 2024-12-03T02:36:00,634 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting c808195946fe41f78a90e08db8ad8b0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733193359252 2024-12-03T02:36:00,634 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54a64c5cbabc4a6197216acbde78a534, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733193358497 2024-12-03T02:36:00,635 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00d95a638efd4c108bddba062b1ec740, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733193359238 2024-12-03T02:36:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:00,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=133.00 KB heapSize=382.24 KB 2024-12-03T02:36:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:00,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:00,665 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:00,665 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/ca3a3b35135e4b7392c9e6d6c76e9e7c is 50, key is test_row_0/B:col10/1733193359888/Put/seqid=0 2024-12-03T02:36:00,680 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T02:36:00,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203008f14e6ada74f1993c775fa0ee8a558_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193360137/Put/seqid=0 2024-12-03T02:36:00,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193420697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,708 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120339892eb3ab504c9497ed63b5181a4704_0be0304c4b25813eaf7af443d2898000 store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:00,710 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120339892eb3ab504c9497ed63b5181a4704_0be0304c4b25813eaf7af443d2898000, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:00,710 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120339892eb3ab504c9497ed63b5181a4704_0be0304c4b25813eaf7af443d2898000 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:00,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742378_1554 (size=12154) 2024-12-03T02:36:00,740 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:00,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742377_1553 (size=12104) 2024-12-03T02:36:00,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742379_1555 (size=4469) 2024-12-03T02:36:00,746 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203008f14e6ada74f1993c775fa0ee8a558_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203008f14e6ada74f1993c775fa0ee8a558_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:00,746 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#A#compaction#467 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:00,747 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/9e8cc9e198044c9aac93366fa6613f72 is 175, key is test_row_0/A:col10/1733193359888/Put/seqid=0 2024-12-03T02:36:00,747 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/bb2aebf3ad3a47009a33986476a3b0c5, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:00,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/bb2aebf3ad3a47009a33986476a3b0c5 is 175, key is test_row_0/A:col10/1733193360137/Put/seqid=0 2024-12-03T02:36:00,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T02:36:00,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:00,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:00,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:00,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:00,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:00,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742381_1557 (size=30955) 2024-12-03T02:36:00,758 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=44.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/bb2aebf3ad3a47009a33986476a3b0c5 2024-12-03T02:36:00,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742380_1556 (size=31058) 2024-12-03T02:36:00,765 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/9e8cc9e198044c9aac93366fa6613f72 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/9e8cc9e198044c9aac93366fa6613f72 2024-12-03T02:36:00,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/fb692df026d74ccca32fe0cda20e80e0 is 50, key is test_row_0/B:col10/1733193360137/Put/seqid=0 2024-12-03T02:36:00,772 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/A of 0be0304c4b25813eaf7af443d2898000 into 9e8cc9e198044c9aac93366fa6613f72(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:00,772 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:00,772 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/A, priority=13, startTime=1733193360628; duration=0sec 2024-12-03T02:36:00,772 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:00,772 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:A 2024-12-03T02:36:00,772 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:00,774 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:00,774 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/C is initiating minor compaction (all files) 2024-12-03T02:36:00,774 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/C in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:00,774 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/64f1f7a3cb094cf0910550c8e188572d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/29b56d7aa4114ba699294a7beeb3101a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/a0390e8b559a4dbd89fcb620a2034b47] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=35.2 K 2024-12-03T02:36:00,775 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64f1f7a3cb094cf0910550c8e188572d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733193358431 2024-12-03T02:36:00,775 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b56d7aa4114ba699294a7beeb3101a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733193358497 2024-12-03T02:36:00,775 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0390e8b559a4dbd89fcb620a2034b47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733193359252 2024-12-03T02:36:00,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193420774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193420775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,785 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#C#compaction#470 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:00,785 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/1c0d121e45e748a780f9557a3cfc4294 is 50, key is test_row_0/C:col10/1733193359888/Put/seqid=0 2024-12-03T02:36:00,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742383_1559 (size=12104) 2024-12-03T02:36:00,791 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/1c0d121e45e748a780f9557a3cfc4294 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1c0d121e45e748a780f9557a3cfc4294 2024-12-03T02:36:00,794 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/C of 0be0304c4b25813eaf7af443d2898000 into 1c0d121e45e748a780f9557a3cfc4294(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:00,794 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:00,795 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/C, priority=13, startTime=1733193360633; duration=0sec 2024-12-03T02:36:00,795 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:00,795 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:C 2024-12-03T02:36:00,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193420805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:00,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742382_1558 (size=12001) 2024-12-03T02:36:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T02:36:00,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T02:36:00,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:00,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:00,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:00,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:00,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193421014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:01,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T02:36:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:01,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:01,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,165 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/ca3a3b35135e4b7392c9e6d6c76e9e7c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/ca3a3b35135e4b7392c9e6d6c76e9e7c 2024-12-03T02:36:01,187 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/B of 0be0304c4b25813eaf7af443d2898000 into ca3a3b35135e4b7392c9e6d6c76e9e7c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:01,187 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:01,188 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/B, priority=13, startTime=1733193360628; duration=0sec 2024-12-03T02:36:01,188 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:01,188 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:B 2024-12-03T02:36:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T02:36:01,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/fb692df026d74ccca32fe0cda20e80e0 2024-12-03T02:36:01,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T02:36:01,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:01,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:01,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:01,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/d3cbc6a436374d57a374b1bf28b81ca3 is 50, key is test_row_0/C:col10/1733193360137/Put/seqid=0 2024-12-03T02:36:01,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742384_1560 (size=12001) 2024-12-03T02:36:01,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/d3cbc6a436374d57a374b1bf28b81ca3 2024-12-03T02:36:01,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193421285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:01,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193421287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:01,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/bb2aebf3ad3a47009a33986476a3b0c5 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/bb2aebf3ad3a47009a33986476a3b0c5 2024-12-03T02:36:01,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193421327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:01,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/bb2aebf3ad3a47009a33986476a3b0c5, entries=150, sequenceid=112, filesize=30.2 K 2024-12-03T02:36:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/fb692df026d74ccca32fe0cda20e80e0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fb692df026d74ccca32fe0cda20e80e0 2024-12-03T02:36:01,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T02:36:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:01,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:01,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fb692df026d74ccca32fe0cda20e80e0, entries=150, sequenceid=112, filesize=11.7 K 2024-12-03T02:36:01,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/d3cbc6a436374d57a374b1bf28b81ca3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3cbc6a436374d57a374b1bf28b81ca3 2024-12-03T02:36:01,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3cbc6a436374d57a374b1bf28b81ca3, entries=150, sequenceid=112, filesize=11.7 K 2024-12-03T02:36:01,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.00 KB/136188, heapSize ~382.20 KB/391368, currentSize=46.96 KB/48090 for 0be0304c4b25813eaf7af443d2898000 in 815ms, sequenceid=112, compaction requested=false 2024-12-03T02:36:01,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:01,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T02:36:01,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:01,540 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T02:36:01,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:01,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:01,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:01,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:01,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:01,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203eebc49c0c532417a8160fdd46dba242f_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193360661/Put/seqid=0 2024-12-03T02:36:01,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742385_1561 (size=12154) 2024-12-03T02:36:01,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:01,618 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203eebc49c0c532417a8160fdd46dba242f_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eebc49c0c532417a8160fdd46dba242f_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/85cd720785b246d8af84133a642bcd9d, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/85cd720785b246d8af84133a642bcd9d is 175, key is test_row_0/A:col10/1733193360661/Put/seqid=0 2024-12-03T02:36:01,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742386_1562 (size=30955) 2024-12-03T02:36:01,654 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=125, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/85cd720785b246d8af84133a642bcd9d 2024-12-03T02:36:01,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/a0ef94733d4f4bf7b3068add276fc38b is 50, key is test_row_0/B:col10/1733193360661/Put/seqid=0 2024-12-03T02:36:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T02:36:01,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742387_1563 (size=12001) 2024-12-03T02:36:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:01,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:01,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193361926, max timestamp=1733193361950Num uniques 859; 2024-12-03T02:36:01,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:36:01,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193361926, max timestamp=1733193361950Num uniques 859; 2024-12-03T02:36:01,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:36:01,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733193361926, max timestamp=1733193361950Num uniques 859; 2024-12-03T02:36:01,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T02:36:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193422068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:02,144 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/a0ef94733d4f4bf7b3068add276fc38b 2024-12-03T02:36:02,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/92e49de1315e4b1ab873e6ac8aeeb6e7 is 50, key is test_row_0/C:col10/1733193360661/Put/seqid=0 2024-12-03T02:36:02,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193422184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:02,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742388_1564 (size=12001) 2024-12-03T02:36:02,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193422307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:02,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193422307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:02,308 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:02,311 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:02,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193422394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:02,590 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/92e49de1315e4b1ab873e6ac8aeeb6e7 2024-12-03T02:36:02,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/85cd720785b246d8af84133a642bcd9d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/85cd720785b246d8af84133a642bcd9d 2024-12-03T02:36:02,624 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/85cd720785b246d8af84133a642bcd9d, entries=150, sequenceid=125, filesize=30.2 K 2024-12-03T02:36:02,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/a0ef94733d4f4bf7b3068add276fc38b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/a0ef94733d4f4bf7b3068add276fc38b 2024-12-03T02:36:02,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193422645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:02,648 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:02,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36054 deadline: 1733193422647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:02,655 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:02,657 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/a0ef94733d4f4bf7b3068add276fc38b, entries=150, sequenceid=125, filesize=11.7 K 2024-12-03T02:36:02,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/92e49de1315e4b1ab873e6ac8aeeb6e7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/92e49de1315e4b1ab873e6ac8aeeb6e7 2024-12-03T02:36:02,707 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/92e49de1315e4b1ab873e6ac8aeeb6e7, entries=150, sequenceid=125, filesize=11.7 K 2024-12-03T02:36:02,708 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0be0304c4b25813eaf7af443d2898000 in 1168ms, sequenceid=125, compaction requested=true 2024-12-03T02:36:02,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:02,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:02,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-03T02:36:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-03T02:36:02,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-03T02:36:02,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-12-03T02:36:02,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.1420 sec 2024-12-03T02:36:02,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733193362011, max timestamp=1733193362712Num uniques 1018; 2024-12-03T02:36:02,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T02:36:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:02,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=102.87 KB heapSize=292.20 KB 2024-12-03T02:36:02,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:02,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:02,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:02,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:36:02,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:02,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T02:36:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T02:36:02,728 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:02,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:36:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-03T02:36:02,730 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:36:02,731 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:36:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T02:36:02,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:36:02,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120329ab8341155d4e7a95dd58c2a66eff6d_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193362064/Put/seqid=0 2024-12-03T02:36:02,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742389_1565 (size=14794) 2024-12-03T02:36:02,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193422820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T02:36:02,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T02:36:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:02,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:02,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:02,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193422935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:02,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:03,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T02:36:03,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:03,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:03,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T02:36:03,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193423145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:03,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:03,182 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120329ab8341155d4e7a95dd58c2a66eff6d_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120329ab8341155d4e7a95dd58c2a66eff6d_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:03,183 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ab789ef9d7cf4b10ba5d1cf31c77cec7, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:03,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ab789ef9d7cf4b10ba5d1cf31c77cec7 is 175, key is test_row_0/A:col10/1733193362064/Put/seqid=0 2024-12-03T02:36:03,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742390_1566 (size=39749) 2024-12-03T02:36:03,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T02:36:03,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:03,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:03,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:03,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ab789ef9d7cf4b10ba5d1cf31c77cec7 2024-12-03T02:36:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:03,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/d085d369080f41d1b5965b09d137cef6 is 50, key is test_row_0/B:col10/1733193362064/Put/seqid=0 2024-12-03T02:36:03,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742391_1567 (size=12151) 2024-12-03T02:36:03,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/d085d369080f41d1b5965b09d137cef6 2024-12-03T02:36:03,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/1e14f52551f24156990c76708a168c98 is 50, key is test_row_0/C:col10/1733193362064/Put/seqid=0 2024-12-03T02:36:03,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T02:36:03,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:03,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:03,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T02:36:03,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742392_1568 (size=12151) 2024-12-03T02:36:03,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/1e14f52551f24156990c76708a168c98 2024-12-03T02:36:03,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ab789ef9d7cf4b10ba5d1cf31c77cec7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ab789ef9d7cf4b10ba5d1cf31c77cec7 2024-12-03T02:36:03,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193423455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:03,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ab789ef9d7cf4b10ba5d1cf31c77cec7, entries=200, sequenceid=163, filesize=38.8 K 2024-12-03T02:36:03,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/d085d369080f41d1b5965b09d137cef6 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d085d369080f41d1b5965b09d137cef6 2024-12-03T02:36:03,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d085d369080f41d1b5965b09d137cef6, entries=150, sequenceid=163, filesize=11.9 K 2024-12-03T02:36:03,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/1e14f52551f24156990c76708a168c98 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1e14f52551f24156990c76708a168c98 2024-12-03T02:36:03,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1e14f52551f24156990c76708a168c98, entries=150, sequenceid=163, filesize=11.9 K 2024-12-03T02:36:03,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~292.16 KB/299168, currentSize=73.80 KB/75570 for 0be0304c4b25813eaf7af443d2898000 in 799ms, sequenceid=163, compaction requested=true 2024-12-03T02:36:03,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:03,517 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:36:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:36:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:36:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:36:03,518 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:36:03,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:03,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T02:36:03,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,520 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T02:36:03,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:03,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:03,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:03,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:03,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:03,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:03,527 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132717 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:36:03,528 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/A is initiating minor compaction (all files) 2024-12-03T02:36:03,528 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/A in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,528 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:36:03,528 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/B is initiating minor compaction (all files) 2024-12-03T02:36:03,528 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/9e8cc9e198044c9aac93366fa6613f72, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/bb2aebf3ad3a47009a33986476a3b0c5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/85cd720785b246d8af84133a642bcd9d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ab789ef9d7cf4b10ba5d1cf31c77cec7] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=129.6 K 2024-12-03T02:36:03,528 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/B in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,528 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,528 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/ca3a3b35135e4b7392c9e6d6c76e9e7c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fb692df026d74ccca32fe0cda20e80e0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/a0ef94733d4f4bf7b3068add276fc38b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d085d369080f41d1b5965b09d137cef6] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=47.1 K 2024-12-03T02:36:03,528 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/9e8cc9e198044c9aac93366fa6613f72, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/bb2aebf3ad3a47009a33986476a3b0c5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/85cd720785b246d8af84133a642bcd9d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ab789ef9d7cf4b10ba5d1cf31c77cec7] 2024-12-03T02:36:03,539 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ca3a3b35135e4b7392c9e6d6c76e9e7c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733193359252 2024-12-03T02:36:03,539 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e8cc9e198044c9aac93366fa6613f72, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733193359252 2024-12-03T02:36:03,543 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb2aebf3ad3a47009a33986476a3b0c5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733193360136 2024-12-03T02:36:03,544 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting fb692df026d74ccca32fe0cda20e80e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733193360136 2024-12-03T02:36:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203315e4f9f6124451695bc26fe2a3d4d24_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193362780/Put/seqid=0 2024-12-03T02:36:03,547 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ef94733d4f4bf7b3068add276fc38b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733193360661 2024-12-03T02:36:03,548 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85cd720785b246d8af84133a642bcd9d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733193360661 2024-12-03T02:36:03,551 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d085d369080f41d1b5965b09d137cef6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733193362011 2024-12-03T02:36:03,552 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab789ef9d7cf4b10ba5d1cf31c77cec7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733193362011 2024-12-03T02:36:03,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742393_1569 (size=12304) 2024-12-03T02:36:03,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:03,624 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203315e4f9f6124451695bc26fe2a3d4d24_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203315e4f9f6124451695bc26fe2a3d4d24_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:03,626 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#B#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:03,626 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/83b652794c1e4f91a6479e4db2bfd244 is 50, key is test_row_0/B:col10/1733193362064/Put/seqid=0 2024-12-03T02:36:03,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/c64564070ad14c778037e311aa79a7f7, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:03,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/c64564070ad14c778037e311aa79a7f7 is 175, key is test_row_0/A:col10/1733193362780/Put/seqid=0 2024-12-03T02:36:03,639 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:03,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742394_1570 (size=12391) 2024-12-03T02:36:03,692 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203eb58cfb4422a4041a7300989c3e5dceb_0be0304c4b25813eaf7af443d2898000 store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:03,694 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203eb58cfb4422a4041a7300989c3e5dceb_0be0304c4b25813eaf7af443d2898000, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:03,694 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203eb58cfb4422a4041a7300989c3e5dceb_0be0304c4b25813eaf7af443d2898000 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:03,705 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/83b652794c1e4f91a6479e4db2bfd244 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/83b652794c1e4f91a6479e4db2bfd244 2024-12-03T02:36:03,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742395_1571 (size=31105) 2024-12-03T02:36:03,738 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/B of 0be0304c4b25813eaf7af443d2898000 into 83b652794c1e4f91a6479e4db2bfd244(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:03,738 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:03,739 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/B, priority=12, startTime=1733193363518; duration=0sec 2024-12-03T02:36:03,739 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:03,739 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:B 2024-12-03T02:36:03,739 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:36:03,742 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:36:03,742 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/C is initiating minor compaction (all files) 2024-12-03T02:36:03,742 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/C in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:03,743 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1c0d121e45e748a780f9557a3cfc4294, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3cbc6a436374d57a374b1bf28b81ca3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/92e49de1315e4b1ab873e6ac8aeeb6e7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1e14f52551f24156990c76708a168c98] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=47.1 K 2024-12-03T02:36:03,745 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c0d121e45e748a780f9557a3cfc4294, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733193359252 2024-12-03T02:36:03,745 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d3cbc6a436374d57a374b1bf28b81ca3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733193360136 2024-12-03T02:36:03,746 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e49de1315e4b1ab873e6ac8aeeb6e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733193360661 2024-12-03T02:36:03,746 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e14f52551f24156990c76708a168c98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733193362011 2024-12-03T02:36:03,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742396_1572 (size=4469) 2024-12-03T02:36:03,758 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#C#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:03,759 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/37ca2f3047fd4af0915d2c80f10ceb6b is 50, key is test_row_0/C:col10/1733193362064/Put/seqid=0 2024-12-03T02:36:03,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742397_1573 (size=12391) 2024-12-03T02:36:03,815 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/37ca2f3047fd4af0915d2c80f10ceb6b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37ca2f3047fd4af0915d2c80f10ceb6b 2024-12-03T02:36:03,829 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/C of 0be0304c4b25813eaf7af443d2898000 into 37ca2f3047fd4af0915d2c80f10ceb6b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:03,829 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:03,829 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/C, priority=12, startTime=1733193363518; duration=0sec 2024-12-03T02:36:03,830 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:03,830 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:C 2024-12-03T02:36:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T02:36:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:03,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193424032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:04,125 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/c64564070ad14c778037e311aa79a7f7 2024-12-03T02:36:04,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/fec0ed01480b466d94daf236b24d84a5 is 50, key is test_row_0/B:col10/1733193362780/Put/seqid=0 2024-12-03T02:36:04,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742398_1574 (size=12151) 2024-12-03T02:36:04,148 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#A#compaction#480 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:04,148 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/99f8d60c2cd94e739e7a2674afb76628 is 175, key is test_row_0/A:col10/1733193362064/Put/seqid=0 2024-12-03T02:36:04,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193424145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:04,152 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/fec0ed01480b466d94daf236b24d84a5 2024-12-03T02:36:04,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742399_1575 (size=31345) 2024-12-03T02:36:04,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/ced6619631dc438192b1b70fa8f17df2 is 50, key is test_row_0/C:col10/1733193362780/Put/seqid=0 2024-12-03T02:36:04,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742400_1576 (size=12151) 2024-12-03T02:36:04,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193424314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:04,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193424315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:04,316 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:04,316 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:04,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193424362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:04,588 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/99f8d60c2cd94e739e7a2674afb76628 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/99f8d60c2cd94e739e7a2674afb76628 2024-12-03T02:36:04,598 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/A of 0be0304c4b25813eaf7af443d2898000 into 99f8d60c2cd94e739e7a2674afb76628(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-12-03T02:36:04,598 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:04,598 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/A, priority=12, startTime=1733193363517; duration=1sec 2024-12-03T02:36:04,598 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:04,598 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:A 2024-12-03T02:36:04,629 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/ced6619631dc438192b1b70fa8f17df2 2024-12-03T02:36:04,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/c64564070ad14c778037e311aa79a7f7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/c64564070ad14c778037e311aa79a7f7 2024-12-03T02:36:04,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193424674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:04,685 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/c64564070ad14c778037e311aa79a7f7, entries=150, sequenceid=177, filesize=30.4 K 2024-12-03T02:36:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/fec0ed01480b466d94daf236b24d84a5 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fec0ed01480b466d94daf236b24d84a5 2024-12-03T02:36:04,735 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fec0ed01480b466d94daf236b24d84a5, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T02:36:04,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/ced6619631dc438192b1b70fa8f17df2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ced6619631dc438192b1b70fa8f17df2 2024-12-03T02:36:04,786 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ced6619631dc438192b1b70fa8f17df2, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T02:36:04,789 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0be0304c4b25813eaf7af443d2898000 in 1268ms, sequenceid=177, compaction requested=false 2024-12-03T02:36:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-03T02:36:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-03T02:36:04,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-03T02:36:04,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0680 sec 2024-12-03T02:36:04,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.0850 sec 2024-12-03T02:36:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T02:36:04,875 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:04,877 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:36:04,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-03T02:36:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T02:36:04,901 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:36:04,915 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:36:04,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:36:05,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T02:36:05,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T02:36:05,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:05,083 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T02:36:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035d31ccac8b4b49e3a44338ea76137e65_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193364022/Put/seqid=0 2024-12-03T02:36:05,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742401_1577 (size=12304) 2024-12-03T02:36:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:05,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T02:36:05,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193425252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:05,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:05,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:05,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:05,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193425372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:05,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T02:36:05,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:05,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193425587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:05,608 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035d31ccac8b4b49e3a44338ea76137e65_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035d31ccac8b4b49e3a44338ea76137e65_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:05,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/aebd8d2f330e4a61ad08f9237026002d, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/aebd8d2f330e4a61ad08f9237026002d is 175, key is test_row_0/A:col10/1733193364022/Put/seqid=0 2024-12-03T02:36:05,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742402_1578 (size=31105) 2024-12-03T02:36:05,691 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/aebd8d2f330e4a61ad08f9237026002d 2024-12-03T02:36:05,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/e4bb14f751854461b3d746d78dc20416 is 50, key is test_row_0/B:col10/1733193364022/Put/seqid=0 2024-12-03T02:36:05,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742403_1579 (size=12151) 2024-12-03T02:36:05,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193425905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T02:36:06,172 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/e4bb14f751854461b3d746d78dc20416 2024-12-03T02:36:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/eb2c26872a294879a0a32f0ae590e849 is 50, key is test_row_0/C:col10/1733193364022/Put/seqid=0 2024-12-03T02:36:06,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742404_1580 (size=12151) 2024-12-03T02:36:06,220 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/eb2c26872a294879a0a32f0ae590e849 2024-12-03T02:36:06,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/aebd8d2f330e4a61ad08f9237026002d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/aebd8d2f330e4a61ad08f9237026002d 2024-12-03T02:36:06,241 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/aebd8d2f330e4a61ad08f9237026002d, entries=150, sequenceid=198, filesize=30.4 K 2024-12-03T02:36:06,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/e4bb14f751854461b3d746d78dc20416 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/e4bb14f751854461b3d746d78dc20416 2024-12-03T02:36:06,252 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/e4bb14f751854461b3d746d78dc20416, entries=150, sequenceid=198, filesize=11.9 K 2024-12-03T02:36:06,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/eb2c26872a294879a0a32f0ae590e849 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/eb2c26872a294879a0a32f0ae590e849 2024-12-03T02:36:06,257 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/eb2c26872a294879a0a32f0ae590e849, entries=150, sequenceid=198, filesize=11.9 K 2024-12-03T02:36:06,258 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0be0304c4b25813eaf7af443d2898000 in 1175ms, sequenceid=198, compaction requested=true 2024-12-03T02:36:06,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:06,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:06,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-03T02:36:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-03T02:36:06,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-03T02:36:06,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-12-03T02:36:06,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.3860 sec 2024-12-03T02:36:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:06,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T02:36:06,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:06,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:06,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:06,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:06,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:06,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:06,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f202a44eca0b476fa89b01352a3f7d32_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193365248/Put/seqid=0 2024-12-03T02:36:06,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742405_1581 (size=12304) 2024-12-03T02:36:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193426499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193426605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:06,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36054 deadline: 1733193426675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:06,678 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:06,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193426695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:06,698 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_0' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:06,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193426815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:06,852 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:06,857 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f202a44eca0b476fa89b01352a3f7d32_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f202a44eca0b476fa89b01352a3f7d32_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:06,858 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/dbd06c7af487458b855d381fee9a9d72, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:06,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/dbd06c7af487458b855d381fee9a9d72 is 175, key is test_row_0/A:col10/1733193365248/Put/seqid=0 2024-12-03T02:36:06,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742406_1582 (size=31105) 2024-12-03T02:36:07,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T02:36:07,055 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:07,055 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:36:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-03T02:36:07,067 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:36:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T02:36:07,067 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:36:07,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:36:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193427125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:07,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T02:36:07,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T02:36:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,284 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/dbd06c7af487458b855d381fee9a9d72 2024-12-03T02:36:07,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/21ea997afa844c3b97ff9878e3798ea5 is 50, key is test_row_0/B:col10/1733193365248/Put/seqid=0 2024-12-03T02:36:07,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742407_1583 (size=12151) 2024-12-03T02:36:07,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T02:36:07,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:07,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T02:36:07,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T02:36:07,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:07,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193427635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:07,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T02:36:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:07,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T02:36:07,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/21ea997afa844c3b97ff9878e3798ea5 2024-12-03T02:36:07,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/352dfb5493f448978ca4e7378c625efa is 50, key is test_row_0/C:col10/1733193365248/Put/seqid=0 2024-12-03T02:36:07,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742408_1584 (size=12151) 2024-12-03T02:36:07,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/352dfb5493f448978ca4e7378c625efa 2024-12-03T02:36:07,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/dbd06c7af487458b855d381fee9a9d72 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dbd06c7af487458b855d381fee9a9d72 2024-12-03T02:36:07,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dbd06c7af487458b855d381fee9a9d72, entries=150, sequenceid=213, filesize=30.4 K 2024-12-03T02:36:07,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/21ea997afa844c3b97ff9878e3798ea5 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/21ea997afa844c3b97ff9878e3798ea5 2024-12-03T02:36:07,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/21ea997afa844c3b97ff9878e3798ea5, entries=150, sequenceid=213, filesize=11.9 K 2024-12-03T02:36:07,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/352dfb5493f448978ca4e7378c625efa as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/352dfb5493f448978ca4e7378c625efa 2024-12-03T02:36:07,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/352dfb5493f448978ca4e7378c625efa, entries=150, sequenceid=213, filesize=11.9 K 2024-12-03T02:36:07,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0be0304c4b25813eaf7af443d2898000 in 1411ms, sequenceid=213, compaction requested=true 2024-12-03T02:36:07,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:07,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:36:07,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:07,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:36:07,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:36:07,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:36:07,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T02:36:07,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T02:36:07,842 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:36:07,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,842 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T02:36:07,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:07,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:07,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:07,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:07,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:07,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:07,843 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:36:07,843 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:36:07,844 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/A is initiating minor compaction (all files) 2024-12-03T02:36:07,844 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/A in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,844 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/99f8d60c2cd94e739e7a2674afb76628, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/c64564070ad14c778037e311aa79a7f7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/aebd8d2f330e4a61ad08f9237026002d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dbd06c7af487458b855d381fee9a9d72] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=121.7 K 2024-12-03T02:36:07,844 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,844 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/99f8d60c2cd94e739e7a2674afb76628, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/c64564070ad14c778037e311aa79a7f7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/aebd8d2f330e4a61ad08f9237026002d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dbd06c7af487458b855d381fee9a9d72] 2024-12-03T02:36:07,845 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99f8d60c2cd94e739e7a2674afb76628, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733193362011 2024-12-03T02:36:07,845 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:36:07,845 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/C is initiating minor compaction (all files) 2024-12-03T02:36:07,845 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/C in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,845 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37ca2f3047fd4af0915d2c80f10ceb6b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ced6619631dc438192b1b70fa8f17df2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/eb2c26872a294879a0a32f0ae590e849, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/352dfb5493f448978ca4e7378c625efa] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=47.7 K 2024-12-03T02:36:07,845 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting c64564070ad14c778037e311aa79a7f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733193362780 2024-12-03T02:36:07,845 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ca2f3047fd4af0915d2c80f10ceb6b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733193362011 2024-12-03T02:36:07,845 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting aebd8d2f330e4a61ad08f9237026002d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733193364018 2024-12-03T02:36:07,846 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ced6619631dc438192b1b70fa8f17df2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733193362780 2024-12-03T02:36:07,846 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbd06c7af487458b855d381fee9a9d72, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733193365238 2024-12-03T02:36:07,846 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting eb2c26872a294879a0a32f0ae590e849, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733193364018 2024-12-03T02:36:07,846 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 352dfb5493f448978ca4e7378c625efa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733193365238 2024-12-03T02:36:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120389d0d51f981c41858eece441da35e174_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193366486/Put/seqid=0 2024-12-03T02:36:07,864 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:07,869 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:07,870 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/8a3349a364064a66aca924c9802e0b4a is 50, key is test_row_0/C:col10/1733193365248/Put/seqid=0 2024-12-03T02:36:07,872 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f5b842a5a6d1426aa63c65bd969435f3_0be0304c4b25813eaf7af443d2898000 store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:07,874 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f5b842a5a6d1426aa63c65bd969435f3_0be0304c4b25813eaf7af443d2898000, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:07,874 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f5b842a5a6d1426aa63c65bd969435f3_0be0304c4b25813eaf7af443d2898000 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:07,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742409_1585 (size=12304) 2024-12-03T02:36:07,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742410_1586 (size=4469) 2024-12-03T02:36:07,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:07,884 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#A#compaction#491 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:07,885 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ecc9fae55d434836bed22b91a0c38381 is 175, key is test_row_0/A:col10/1733193365248/Put/seqid=0 2024-12-03T02:36:07,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742411_1587 (size=12527) 2024-12-03T02:36:07,909 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120389d0d51f981c41858eece441da35e174_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389d0d51f981c41858eece441da35e174_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:07,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742412_1588 (size=31481) 2024-12-03T02:36:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/3f75c853421e4e86b63b2b57feb9fc59, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/3f75c853421e4e86b63b2b57feb9fc59 is 175, key is test_row_0/A:col10/1733193366486/Put/seqid=0 2024-12-03T02:36:07,930 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/8a3349a364064a66aca924c9802e0b4a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/8a3349a364064a66aca924c9802e0b4a 2024-12-03T02:36:07,935 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ecc9fae55d434836bed22b91a0c38381 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ecc9fae55d434836bed22b91a0c38381 2024-12-03T02:36:07,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742413_1589 (size=31105) 2024-12-03T02:36:07,951 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/C of 0be0304c4b25813eaf7af443d2898000 into 8a3349a364064a66aca924c9802e0b4a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:07,951 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:07,951 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/C, priority=12, startTime=1733193367842; duration=0sec 2024-12-03T02:36:07,952 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:07,952 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:C 2024-12-03T02:36:07,952 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/A of 0be0304c4b25813eaf7af443d2898000 into ecc9fae55d434836bed22b91a0c38381(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:07,952 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:36:07,952 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:07,952 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/A, priority=12, startTime=1733193367842; duration=0sec 2024-12-03T02:36:07,952 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:07,952 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:A 2024-12-03T02:36:07,956 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/3f75c853421e4e86b63b2b57feb9fc59 2024-12-03T02:36:07,967 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:36:07,967 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/B is initiating minor compaction (all files) 2024-12-03T02:36:07,967 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/B in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:07,967 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/83b652794c1e4f91a6479e4db2bfd244, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fec0ed01480b466d94daf236b24d84a5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/e4bb14f751854461b3d746d78dc20416, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/21ea997afa844c3b97ff9878e3798ea5] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=47.7 K 2024-12-03T02:36:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/8300ace2d13341a39c0441bb0e6a9002 is 50, key is test_row_0/B:col10/1733193366486/Put/seqid=0 2024-12-03T02:36:07,971 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b652794c1e4f91a6479e4db2bfd244, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733193362011 2024-12-03T02:36:07,975 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting fec0ed01480b466d94daf236b24d84a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733193362780 2024-12-03T02:36:07,979 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e4bb14f751854461b3d746d78dc20416, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733193364018 2024-12-03T02:36:07,983 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 21ea997afa844c3b97ff9878e3798ea5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733193365238 2024-12-03T02:36:07,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742414_1590 (size=12151) 2024-12-03T02:36:07,988 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/8300ace2d13341a39c0441bb0e6a9002 2024-12-03T02:36:08,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/c991c64b964f4db9b0297dcfb095be57 is 50, key is test_row_0/C:col10/1733193366486/Put/seqid=0 2024-12-03T02:36:08,009 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:08,010 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/7677581d113d4edcb36f3b1db6ea2459 is 50, key is test_row_0/B:col10/1733193365248/Put/seqid=0 2024-12-03T02:36:08,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742415_1591 (size=12151) 2024-12-03T02:36:08,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742416_1592 (size=12527) 2024-12-03T02:36:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T02:36:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:08,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:08,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193428369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193428373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,412 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/c991c64b964f4db9b0297dcfb095be57 2024-12-03T02:36:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/3f75c853421e4e86b63b2b57feb9fc59 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3f75c853421e4e86b63b2b57feb9fc59 2024-12-03T02:36:08,423 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3f75c853421e4e86b63b2b57feb9fc59, entries=150, sequenceid=230, filesize=30.4 K 2024-12-03T02:36:08,423 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/7677581d113d4edcb36f3b1db6ea2459 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/7677581d113d4edcb36f3b1db6ea2459 2024-12-03T02:36:08,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/8300ace2d13341a39c0441bb0e6a9002 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8300ace2d13341a39c0441bb0e6a9002 2024-12-03T02:36:08,427 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8300ace2d13341a39c0441bb0e6a9002, entries=150, sequenceid=230, filesize=11.9 K 2024-12-03T02:36:08,428 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/B of 0be0304c4b25813eaf7af443d2898000 into 7677581d113d4edcb36f3b1db6ea2459(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:08,428 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:08,428 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/B, priority=12, startTime=1733193367842; duration=0sec 2024-12-03T02:36:08,428 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:08,428 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:B 2024-12-03T02:36:08,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/c991c64b964f4db9b0297dcfb095be57 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/c991c64b964f4db9b0297dcfb095be57 2024-12-03T02:36:08,433 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/c991c64b964f4db9b0297dcfb095be57, entries=150, sequenceid=230, filesize=11.9 K 2024-12-03T02:36:08,434 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0be0304c4b25813eaf7af443d2898000 in 592ms, sequenceid=230, compaction requested=false 2024-12-03T02:36:08,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:08,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:08,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-03T02:36:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-03T02:36:08,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-03T02:36:08,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3680 sec 2024-12-03T02:36:08,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.3840 sec 2024-12-03T02:36:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:08,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T02:36:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:08,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:08,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b551f0f620454e4583b26446fe13d23e_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193368474/Put/seqid=0 2024-12-03T02:36:08,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742417_1593 (size=14794) 2024-12-03T02:36:08,499 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:08,503 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b551f0f620454e4583b26446fe13d23e_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b551f0f620454e4583b26446fe13d23e_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:08,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/0a24b38d796d4ffea78fd4cd6b35c3e2, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:08,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/0a24b38d796d4ffea78fd4cd6b35c3e2 is 175, key is test_row_0/A:col10/1733193368474/Put/seqid=0 2024-12-03T02:36:08,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742418_1594 (size=39749) 2024-12-03T02:36:08,509 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/0a24b38d796d4ffea78fd4cd6b35c3e2 2024-12-03T02:36:08,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193428507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/858ae92b8c6e4439be3055efe83958db is 50, key is test_row_0/B:col10/1733193368474/Put/seqid=0 2024-12-03T02:36:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193428512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742419_1595 (size=12151) 2024-12-03T02:36:08,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/858ae92b8c6e4439be3055efe83958db 2024-12-03T02:36:08,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/9f4b4cc553e241228cdaea47a92bb52e is 50, key is test_row_0/C:col10/1733193368474/Put/seqid=0 2024-12-03T02:36:08,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742420_1596 (size=12151) 2024-12-03T02:36:08,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/9f4b4cc553e241228cdaea47a92bb52e 2024-12-03T02:36:08,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/0a24b38d796d4ffea78fd4cd6b35c3e2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/0a24b38d796d4ffea78fd4cd6b35c3e2 2024-12-03T02:36:08,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/0a24b38d796d4ffea78fd4cd6b35c3e2, entries=200, sequenceid=249, filesize=38.8 K 2024-12-03T02:36:08,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/858ae92b8c6e4439be3055efe83958db as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/858ae92b8c6e4439be3055efe83958db 2024-12-03T02:36:08,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/858ae92b8c6e4439be3055efe83958db, entries=150, sequenceid=249, filesize=11.9 K 2024-12-03T02:36:08,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/9f4b4cc553e241228cdaea47a92bb52e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f4b4cc553e241228cdaea47a92bb52e 2024-12-03T02:36:08,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f4b4cc553e241228cdaea47a92bb52e, entries=150, sequenceid=249, filesize=11.9 K 2024-12-03T02:36:08,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0be0304c4b25813eaf7af443d2898000 in 122ms, sequenceid=249, compaction requested=true 2024-12-03T02:36:08,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:08,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:36:08,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:08,597 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:08,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:36:08,597 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:08,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:08,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:36:08,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:08,598 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:08,598 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:08,598 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/B is initiating minor compaction (all files) 2024-12-03T02:36:08,598 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/A is initiating minor compaction (all files) 2024-12-03T02:36:08,598 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/B in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:08,598 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/A in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:08,598 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/7677581d113d4edcb36f3b1db6ea2459, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8300ace2d13341a39c0441bb0e6a9002, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/858ae92b8c6e4439be3055efe83958db] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.0 K 2024-12-03T02:36:08,598 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ecc9fae55d434836bed22b91a0c38381, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3f75c853421e4e86b63b2b57feb9fc59, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/0a24b38d796d4ffea78fd4cd6b35c3e2] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=99.9 K 2024-12-03T02:36:08,598 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:08,599 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ecc9fae55d434836bed22b91a0c38381, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3f75c853421e4e86b63b2b57feb9fc59, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/0a24b38d796d4ffea78fd4cd6b35c3e2] 2024-12-03T02:36:08,599 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc9fae55d434836bed22b91a0c38381, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733193365238 2024-12-03T02:36:08,599 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7677581d113d4edcb36f3b1db6ea2459, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733193365238 2024-12-03T02:36:08,599 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f75c853421e4e86b63b2b57feb9fc59, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733193366477 2024-12-03T02:36:08,599 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 8300ace2d13341a39c0441bb0e6a9002, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733193366477 2024-12-03T02:36:08,599 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a24b38d796d4ffea78fd4cd6b35c3e2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733193368351 2024-12-03T02:36:08,599 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 858ae92b8c6e4439be3055efe83958db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733193368366 2024-12-03T02:36:08,606 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:08,607 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/731b8cb76b5842e2a05aeae76cd50f73 is 50, key is test_row_0/B:col10/1733193368474/Put/seqid=0 2024-12-03T02:36:08,608 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:08,612 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203d644741a43894656bb0a4dc9c1a9b98a_0be0304c4b25813eaf7af443d2898000 store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:08,613 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203d644741a43894656bb0a4dc9c1a9b98a_0be0304c4b25813eaf7af443d2898000, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:08,614 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d644741a43894656bb0a4dc9c1a9b98a_0be0304c4b25813eaf7af443d2898000 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:08,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T02:36:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:08,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:08,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:08,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:08,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:08,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:08,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:08,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742421_1597 (size=12629) 2024-12-03T02:36:08,636 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/731b8cb76b5842e2a05aeae76cd50f73 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/731b8cb76b5842e2a05aeae76cd50f73 2024-12-03T02:36:08,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742422_1598 (size=4469) 2024-12-03T02:36:08,643 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#A#compaction#500 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:08,644 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/7068d999eee74c9dac880eddfc1ef96b is 175, key is test_row_0/A:col10/1733193368474/Put/seqid=0 2024-12-03T02:36:08,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034132e6f4c6904942b90e361dab6e4c45_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193368628/Put/seqid=0 2024-12-03T02:36:08,649 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/B of 0be0304c4b25813eaf7af443d2898000 into 731b8cb76b5842e2a05aeae76cd50f73(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:08,649 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:08,649 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/B, priority=13, startTime=1733193368597; duration=0sec 2024-12-03T02:36:08,650 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:08,650 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:B 2024-12-03T02:36:08,650 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:08,655 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:08,655 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/C is initiating minor compaction (all files) 2024-12-03T02:36:08,655 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/C in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:08,655 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/8a3349a364064a66aca924c9802e0b4a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/c991c64b964f4db9b0297dcfb095be57, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f4b4cc553e241228cdaea47a92bb52e] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.0 K 2024-12-03T02:36:08,656 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a3349a364064a66aca924c9802e0b4a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733193365238 2024-12-03T02:36:08,656 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting c991c64b964f4db9b0297dcfb095be57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733193366477 2024-12-03T02:36:08,657 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f4b4cc553e241228cdaea47a92bb52e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733193368366 2024-12-03T02:36:08,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742424_1600 (size=17484) 2024-12-03T02:36:08,664 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#C#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:08,664 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/7ee44e8caf684eac9d473eaa91a88913 is 50, key is test_row_0/C:col10/1733193368474/Put/seqid=0 2024-12-03T02:36:08,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193428660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193428662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193428663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742423_1599 (size=31583) 2024-12-03T02:36:08,680 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/7068d999eee74c9dac880eddfc1ef96b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7068d999eee74c9dac880eddfc1ef96b 2024-12-03T02:36:08,687 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/A of 0be0304c4b25813eaf7af443d2898000 into 7068d999eee74c9dac880eddfc1ef96b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:08,687 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:08,687 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/A, priority=13, startTime=1733193368597; duration=0sec 2024-12-03T02:36:08,687 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:08,687 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:A 2024-12-03T02:36:08,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742425_1601 (size=12629) 2024-12-03T02:36:08,695 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:08,695 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/7ee44e8caf684eac9d473eaa91a88913 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7ee44e8caf684eac9d473eaa91a88913 2024-12-03T02:36:08,710 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034132e6f4c6904942b90e361dab6e4c45_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034132e6f4c6904942b90e361dab6e4c45_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:08,716 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ea7f566b520c4d23a7d7aedcfee151b0, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:08,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ea7f566b520c4d23a7d7aedcfee151b0 is 175, key is test_row_0/A:col10/1733193368628/Put/seqid=0 2024-12-03T02:36:08,727 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/C of 0be0304c4b25813eaf7af443d2898000 into 7ee44e8caf684eac9d473eaa91a88913(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:08,727 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:08,727 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/C, priority=13, startTime=1733193368597; duration=0sec 2024-12-03T02:36:08,727 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:08,727 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:C 2024-12-03T02:36:08,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742426_1602 (size=48589) 2024-12-03T02:36:08,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193428774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193428774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193428775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193428984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193428984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:08,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193428985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:09,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ea7f566b520c4d23a7d7aedcfee151b0 2024-12-03T02:36:09,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/b187bd7ebc5a4e5bb779bb23419a045f is 50, key is test_row_0/B:col10/1733193368628/Put/seqid=0 2024-12-03T02:36:09,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742427_1603 (size=12301) 2024-12-03T02:36:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T02:36:09,225 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:09,225 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:36:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-03T02:36:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T02:36:09,227 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:36:09,228 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:36:09,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:36:09,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193429295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:09,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193429295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193429295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T02:36:09,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T02:36:09,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:09,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T02:36:09,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:09,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T02:36:09,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/b187bd7ebc5a4e5bb779bb23419a045f 2024-12-03T02:36:09,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/afce4b9f4d4e46b79d4fc99fd896e127 is 50, key is test_row_0/C:col10/1733193368628/Put/seqid=0 2024-12-03T02:36:09,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742428_1604 (size=12301) 2024-12-03T02:36:09,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T02:36:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:09,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193429805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:09,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193429805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:09,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193429805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:09,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T02:36:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T02:36:09,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/afce4b9f4d4e46b79d4fc99fd896e127 2024-12-03T02:36:09,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ea7f566b520c4d23a7d7aedcfee151b0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ea7f566b520c4d23a7d7aedcfee151b0 2024-12-03T02:36:09,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T02:36:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:09,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:10,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ea7f566b520c4d23a7d7aedcfee151b0, entries=250, sequenceid=267, filesize=47.5 K 2024-12-03T02:36:10,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/b187bd7ebc5a4e5bb779bb23419a045f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b187bd7ebc5a4e5bb779bb23419a045f 2024-12-03T02:36:10,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b187bd7ebc5a4e5bb779bb23419a045f, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T02:36:10,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/afce4b9f4d4e46b79d4fc99fd896e127 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/afce4b9f4d4e46b79d4fc99fd896e127 2024-12-03T02:36:10,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/afce4b9f4d4e46b79d4fc99fd896e127, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T02:36:10,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0be0304c4b25813eaf7af443d2898000 in 1381ms, sequenceid=267, compaction requested=false 2024-12-03T02:36:10,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:10,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T02:36:10,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:10,153 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T02:36:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:10,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e437fef14b854e82834b6c4e9c463cd1_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193368653/Put/seqid=0 2024-12-03T02:36:10,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742429_1605 (size=12454) 2024-12-03T02:36:10,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:10,170 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e437fef14b854e82834b6c4e9c463cd1_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e437fef14b854e82834b6c4e9c463cd1_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:10,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/eaa124b713644c4ea579606983e37155, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:10,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/eaa124b713644c4ea579606983e37155 is 175, key is test_row_0/A:col10/1733193368653/Put/seqid=0 2024-12-03T02:36:10,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742430_1606 (size=31255) 2024-12-03T02:36:10,185 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/eaa124b713644c4ea579606983e37155 2024-12-03T02:36:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/c758b742e2684aadb473de2ce9145812 is 50, key is test_row_0/B:col10/1733193368653/Put/seqid=0 2024-12-03T02:36:10,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742431_1607 (size=12301) 2024-12-03T02:36:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T02:36:10,601 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/c758b742e2684aadb473de2ce9145812 2024-12-03T02:36:10,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/fde52fabc5f74749b7326ba35dbfef60 is 50, key is test_row_0/C:col10/1733193368653/Put/seqid=0 2024-12-03T02:36:10,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742432_1608 (size=12301) 2024-12-03T02:36:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:10,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:10,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193430843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:10,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193430845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:10,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193430849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193430954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193430954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193430954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,025 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/fde52fabc5f74749b7326ba35dbfef60 2024-12-03T02:36:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/eaa124b713644c4ea579606983e37155 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/eaa124b713644c4ea579606983e37155 2024-12-03T02:36:11,032 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/eaa124b713644c4ea579606983e37155, entries=150, sequenceid=284, filesize=30.5 K 2024-12-03T02:36:11,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/c758b742e2684aadb473de2ce9145812 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c758b742e2684aadb473de2ce9145812 2024-12-03T02:36:11,043 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c758b742e2684aadb473de2ce9145812, entries=150, sequenceid=284, filesize=12.0 K 2024-12-03T02:36:11,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/fde52fabc5f74749b7326ba35dbfef60 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/fde52fabc5f74749b7326ba35dbfef60 2024-12-03T02:36:11,049 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/fde52fabc5f74749b7326ba35dbfef60, entries=150, sequenceid=284, filesize=12.0 K 2024-12-03T02:36:11,050 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0be0304c4b25813eaf7af443d2898000 in 897ms, sequenceid=284, compaction requested=true 2024-12-03T02:36:11,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:11,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:11,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-03T02:36:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-03T02:36:11,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-03T02:36:11,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8270 sec 2024-12-03T02:36:11,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8340 sec 2024-12-03T02:36:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:11,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T02:36:11,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:11,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:11,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:11,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:11,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:11,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:11,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c972032648e1424fa3b7ed9f82aa194b_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193371166/Put/seqid=0 2024-12-03T02:36:11,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193431186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193431186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193431186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742433_1609 (size=14994) 2024-12-03T02:36:11,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193431295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193431295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193431305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T02:36:11,375 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:11,375 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:36:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-03T02:36:11,377 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:36:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T02:36:11,377 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:36:11,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:36:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T02:36:11,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193431505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193431505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193431514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T02:36:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:11,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:11,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:11,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:11,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c972032648e1424fa3b7ed9f82aa194b_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c972032648e1424fa3b7ed9f82aa194b_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:11,603 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/f52c5803b0354a37968cadcc636c73b1, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:11,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/f52c5803b0354a37968cadcc636c73b1 is 175, key is test_row_0/A:col10/1733193371166/Put/seqid=0 2024-12-03T02:36:11,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742434_1610 (size=39949) 2024-12-03T02:36:11,607 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/f52c5803b0354a37968cadcc636c73b1 2024-12-03T02:36:11,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/b14051911fb04e3685a77c841ef4036b is 50, key is test_row_0/B:col10/1733193371166/Put/seqid=0 2024-12-03T02:36:11,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742435_1611 (size=12301) 2024-12-03T02:36:11,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T02:36:11,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:11,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:11,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:11,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T02:36:11,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193431815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193431815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193431828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:11,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T02:36:11,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:11,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:11,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:11,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:11,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:11,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T02:36:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:11,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T02:36:12,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/b14051911fb04e3685a77c841ef4036b 2024-12-03T02:36:12,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/7f3aa8567f0346f3b63d8daa170d9dd2 is 50, key is test_row_0/C:col10/1733193371166/Put/seqid=0 2024-12-03T02:36:12,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742436_1612 (size=12301) 2024-12-03T02:36:12,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/7f3aa8567f0346f3b63d8daa170d9dd2 2024-12-03T02:36:12,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/f52c5803b0354a37968cadcc636c73b1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/f52c5803b0354a37968cadcc636c73b1 2024-12-03T02:36:12,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/f52c5803b0354a37968cadcc636c73b1, entries=200, sequenceid=304, filesize=39.0 K 2024-12-03T02:36:12,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/b14051911fb04e3685a77c841ef4036b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b14051911fb04e3685a77c841ef4036b 2024-12-03T02:36:12,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b14051911fb04e3685a77c841ef4036b, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T02:36:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/7f3aa8567f0346f3b63d8daa170d9dd2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7f3aa8567f0346f3b63d8daa170d9dd2 2024-12-03T02:36:12,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7f3aa8567f0346f3b63d8daa170d9dd2, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T02:36:12,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 0be0304c4b25813eaf7af443d2898000 in 982ms, sequenceid=304, compaction requested=true 2024-12-03T02:36:12,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:12,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:36:12,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:12,149 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:36:12,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:36:12,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:12,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:36:12,149 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:36:12,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:12,150 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151376 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:36:12,150 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/A is initiating minor compaction (all files) 2024-12-03T02:36:12,150 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/A in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,151 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7068d999eee74c9dac880eddfc1ef96b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ea7f566b520c4d23a7d7aedcfee151b0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/eaa124b713644c4ea579606983e37155, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/f52c5803b0354a37968cadcc636c73b1] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=147.8 K 2024-12-03T02:36:12,151 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,151 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7068d999eee74c9dac880eddfc1ef96b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ea7f566b520c4d23a7d7aedcfee151b0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/eaa124b713644c4ea579606983e37155, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/f52c5803b0354a37968cadcc636c73b1] 2024-12-03T02:36:12,152 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:36:12,152 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/B is initiating minor compaction (all files) 2024-12-03T02:36:12,152 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/B in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,152 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/731b8cb76b5842e2a05aeae76cd50f73, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b187bd7ebc5a4e5bb779bb23419a045f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c758b742e2684aadb473de2ce9145812, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b14051911fb04e3685a77c841ef4036b] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=48.4 K 2024-12-03T02:36:12,152 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7068d999eee74c9dac880eddfc1ef96b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733193368366 2024-12-03T02:36:12,152 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea7f566b520c4d23a7d7aedcfee151b0, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733193368485 2024-12-03T02:36:12,153 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 731b8cb76b5842e2a05aeae76cd50f73, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733193368366 2024-12-03T02:36:12,153 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b187bd7ebc5a4e5bb779bb23419a045f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733193368491 2024-12-03T02:36:12,153 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaa124b713644c4ea579606983e37155, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733193368653 2024-12-03T02:36:12,154 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting f52c5803b0354a37968cadcc636c73b1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733193370839 2024-12-03T02:36:12,154 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting c758b742e2684aadb473de2ce9145812, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733193368653 2024-12-03T02:36:12,154 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b14051911fb04e3685a77c841ef4036b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733193370847 2024-12-03T02:36:12,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T02:36:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,170 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T02:36:12,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:12,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:12,174 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:12,198 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#B#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:12,203 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/f21c611e3e074c4a9844390a89976f2b is 50, key is test_row_0/B:col10/1733193371166/Put/seqid=0 2024-12-03T02:36:12,204 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c5c119000db040f3b571d14217acd236_0be0304c4b25813eaf7af443d2898000 store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:12,211 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c5c119000db040f3b571d14217acd236_0be0304c4b25813eaf7af443d2898000, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:12,211 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c5c119000db040f3b571d14217acd236_0be0304c4b25813eaf7af443d2898000 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:12,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e9c7c20411ec4b55a433e3fb733214c2_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193371184/Put/seqid=0 2024-12-03T02:36:12,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742437_1613 (size=12915) 2024-12-03T02:36:12,254 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/f21c611e3e074c4a9844390a89976f2b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/f21c611e3e074c4a9844390a89976f2b 2024-12-03T02:36:12,267 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/B of 0be0304c4b25813eaf7af443d2898000 into f21c611e3e074c4a9844390a89976f2b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:12,267 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:12,267 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/B, priority=12, startTime=1733193372149; duration=0sec 2024-12-03T02:36:12,268 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:12,268 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:B 2024-12-03T02:36:12,268 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T02:36:12,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742438_1614 (size=4469) 2024-12-03T02:36:12,271 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T02:36:12,271 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#A#compaction#511 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:12,271 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/C is initiating minor compaction (all files) 2024-12-03T02:36:12,271 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/C in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,271 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7ee44e8caf684eac9d473eaa91a88913, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/afce4b9f4d4e46b79d4fc99fd896e127, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/fde52fabc5f74749b7326ba35dbfef60, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7f3aa8567f0346f3b63d8daa170d9dd2] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=48.4 K 2024-12-03T02:36:12,272 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/abced3de92504d3f84a6d4b87c39cca1 is 175, key is test_row_0/A:col10/1733193371166/Put/seqid=0 2024-12-03T02:36:12,274 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ee44e8caf684eac9d473eaa91a88913, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733193368366 2024-12-03T02:36:12,274 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting afce4b9f4d4e46b79d4fc99fd896e127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733193368491 2024-12-03T02:36:12,275 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting fde52fabc5f74749b7326ba35dbfef60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733193368653 2024-12-03T02:36:12,275 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f3aa8567f0346f3b63d8daa170d9dd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733193370847 2024-12-03T02:36:12,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742439_1615 (size=12454) 2024-12-03T02:36:12,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:12,300 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#C#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:12,301 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/76b29038d3884e729fbd74997c161d8f is 50, key is test_row_0/C:col10/1733193371166/Put/seqid=0 2024-12-03T02:36:12,302 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e9c7c20411ec4b55a433e3fb733214c2_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e9c7c20411ec4b55a433e3fb733214c2_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/e1cfc3cfea154aca89313e3b35f2dca7, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/e1cfc3cfea154aca89313e3b35f2dca7 is 175, key is test_row_0/A:col10/1733193371184/Put/seqid=0 2024-12-03T02:36:12,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742440_1616 (size=31869) 2024-12-03T02:36:12,317 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/abced3de92504d3f84a6d4b87c39cca1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/abced3de92504d3f84a6d4b87c39cca1 2024-12-03T02:36:12,336 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/A of 0be0304c4b25813eaf7af443d2898000 into abced3de92504d3f84a6d4b87c39cca1(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:12,336 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:12,336 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/A, priority=12, startTime=1733193372149; duration=0sec 2024-12-03T02:36:12,336 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:12,336 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:A 2024-12-03T02:36:12,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742442_1618 (size=31255) 2024-12-03T02:36:12,341 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/e1cfc3cfea154aca89313e3b35f2dca7 2024-12-03T02:36:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:12,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:12,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/2f5b1d863c524ee1970a6110ace2fde3 is 50, key is test_row_0/B:col10/1733193371184/Put/seqid=0 2024-12-03T02:36:12,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742441_1617 (size=12915) 2024-12-03T02:36:12,375 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/76b29038d3884e729fbd74997c161d8f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/76b29038d3884e729fbd74997c161d8f 2024-12-03T02:36:12,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742443_1619 (size=12301) 2024-12-03T02:36:12,391 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/C of 0be0304c4b25813eaf7af443d2898000 into 76b29038d3884e729fbd74997c161d8f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:12,391 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:12,391 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/C, priority=12, startTime=1733193372149; duration=0sec 2024-12-03T02:36:12,397 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:12,397 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:C 2024-12-03T02:36:12,399 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/2f5b1d863c524ee1970a6110ace2fde3 2024-12-03T02:36:12,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/ae33f2d0fe5241728f9598b418cd8784 is 50, key is test_row_0/C:col10/1733193371184/Put/seqid=0 2024-12-03T02:36:12,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742444_1620 (size=12301) 2024-12-03T02:36:12,449 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/ae33f2d0fe5241728f9598b418cd8784 2024-12-03T02:36:12,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193432443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193432447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193432451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/e1cfc3cfea154aca89313e3b35f2dca7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/e1cfc3cfea154aca89313e3b35f2dca7 2024-12-03T02:36:12,462 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/e1cfc3cfea154aca89313e3b35f2dca7, entries=150, sequenceid=318, filesize=30.5 K 2024-12-03T02:36:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/2f5b1d863c524ee1970a6110ace2fde3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2f5b1d863c524ee1970a6110ace2fde3 2024-12-03T02:36:12,470 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2f5b1d863c524ee1970a6110ace2fde3, entries=150, sequenceid=318, filesize=12.0 K 2024-12-03T02:36:12,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/ae33f2d0fe5241728f9598b418cd8784 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ae33f2d0fe5241728f9598b418cd8784 2024-12-03T02:36:12,477 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ae33f2d0fe5241728f9598b418cd8784, entries=150, sequenceid=318, filesize=12.0 K 2024-12-03T02:36:12,480 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 0be0304c4b25813eaf7af443d2898000 in 310ms, sequenceid=318, compaction requested=false 2024-12-03T02:36:12,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:12,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-03T02:36:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-03T02:36:12,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-03T02:36:12,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1160 sec 2024-12-03T02:36:12,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.1240 sec 2024-12-03T02:36:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T02:36:12,515 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:12,516 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:36:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-03T02:36:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T02:36:12,527 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:36:12,528 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:36:12,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:36:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:12,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1733193372380, max timestamp=1733193372571Num uniques 859; 2024-12-03T02:36:12,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T02:36:12,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/64035cbb0b46:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T02:36:12,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:12,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:12,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:12,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:12,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:12,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:12,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030b4f7cca6c64422c9de58f83d803b5d7_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193372568/Put/seqid=0 2024-12-03T02:36:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T02:36:12,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193432623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193432632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193432632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742445_1621 (size=12454) 2024-12-03T02:36:12,668 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:12,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T02:36:12,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:12,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,689 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030b4f7cca6c64422c9de58f83d803b5d7_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b4f7cca6c64422c9de58f83d803b5d7_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:12,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/8c9f4a7fe69a4f78b7f16f1339681ac9, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:12,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/8c9f4a7fe69a4f78b7f16f1339681ac9 is 175, key is test_row_0/A:col10/1733193372568/Put/seqid=0 2024-12-03T02:36:12,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742446_1622 (size=31255) 2024-12-03T02:36:12,747 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/8c9f4a7fe69a4f78b7f16f1339681ac9 2024-12-03T02:36:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193432745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193432745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193432756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/484469bc5b3449dd8baba8f517bfc04a is 50, key is test_row_0/B:col10/1733193372568/Put/seqid=0 2024-12-03T02:36:12,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T02:36:12,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:12,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T02:36:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742447_1623 (size=12301) 2024-12-03T02:36:12,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/484469bc5b3449dd8baba8f517bfc04a 2024-12-03T02:36:12,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/d3b4a6d6b69547d196263a724db29d4f is 50, key is test_row_0/C:col10/1733193372568/Put/seqid=0 2024-12-03T02:36:12,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742448_1624 (size=12301) 2024-12-03T02:36:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193432956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193432957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193432975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:12,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T02:36:12,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:12,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:12,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:13,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T02:36:13,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:13,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:13,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:13,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:13,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:13,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T02:36:13,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193433265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:13,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193433265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:13,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193433284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:13,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T02:36:13,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:13,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:13,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:13,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:13,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:13,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/d3b4a6d6b69547d196263a724db29d4f 2024-12-03T02:36:13,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/8c9f4a7fe69a4f78b7f16f1339681ac9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/8c9f4a7fe69a4f78b7f16f1339681ac9 2024-12-03T02:36:13,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/8c9f4a7fe69a4f78b7f16f1339681ac9, entries=150, sequenceid=343, filesize=30.5 K 2024-12-03T02:36:13,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/484469bc5b3449dd8baba8f517bfc04a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/484469bc5b3449dd8baba8f517bfc04a 2024-12-03T02:36:13,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/484469bc5b3449dd8baba8f517bfc04a, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T02:36:13,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/d3b4a6d6b69547d196263a724db29d4f as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3b4a6d6b69547d196263a724db29d4f 2024-12-03T02:36:13,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3b4a6d6b69547d196263a724db29d4f, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T02:36:13,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~98.40 KB/100760, heapSize ~287.06 KB/293952, currentSize=80.51 KB/82440 for 0be0304c4b25813eaf7af443d2898000 in 803ms, sequenceid=343, compaction requested=true 2024-12-03T02:36:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:36:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:13,382 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:36:13,382 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:36:13,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:13,383 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:13,383 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/B is initiating minor compaction (all files) 2024-12-03T02:36:13,383 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/B in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:13,383 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/f21c611e3e074c4a9844390a89976f2b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2f5b1d863c524ee1970a6110ace2fde3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/484469bc5b3449dd8baba8f517bfc04a] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.6 K 2024-12-03T02:36:13,384 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:13,384 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/A is initiating minor compaction (all files) 2024-12-03T02:36:13,384 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/A in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:13,384 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/abced3de92504d3f84a6d4b87c39cca1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/e1cfc3cfea154aca89313e3b35f2dca7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/8c9f4a7fe69a4f78b7f16f1339681ac9] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=92.2 K 2024-12-03T02:36:13,384 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:13,384 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/abced3de92504d3f84a6d4b87c39cca1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/e1cfc3cfea154aca89313e3b35f2dca7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/8c9f4a7fe69a4f78b7f16f1339681ac9] 2024-12-03T02:36:13,384 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting f21c611e3e074c4a9844390a89976f2b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733193370847 2024-12-03T02:36:13,384 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting abced3de92504d3f84a6d4b87c39cca1, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733193370847 2024-12-03T02:36:13,384 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f5b1d863c524ee1970a6110ace2fde3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733193371183 2024-12-03T02:36:13,385 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 484469bc5b3449dd8baba8f517bfc04a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193372568 2024-12-03T02:36:13,385 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1cfc3cfea154aca89313e3b35f2dca7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733193371183 2024-12-03T02:36:13,385 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c9f4a7fe69a4f78b7f16f1339681ac9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193372568 2024-12-03T02:36:13,399 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:13,403 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#B#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:13,404 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/d4660dfc4e3648eab86728f09ac3d5e3 is 50, key is test_row_0/B:col10/1733193372568/Put/seqid=0 2024-12-03T02:36:13,406 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412034cab18cb16a348d0b8054421459e2dba_0be0304c4b25813eaf7af443d2898000 store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:13,407 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412034cab18cb16a348d0b8054421459e2dba_0be0304c4b25813eaf7af443d2898000, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:13,407 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034cab18cb16a348d0b8054421459e2dba_0be0304c4b25813eaf7af443d2898000 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:13,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742449_1625 (size=13017) 2024-12-03T02:36:13,418 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/d4660dfc4e3648eab86728f09ac3d5e3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d4660dfc4e3648eab86728f09ac3d5e3 2024-12-03T02:36:13,422 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/B of 0be0304c4b25813eaf7af443d2898000 into d4660dfc4e3648eab86728f09ac3d5e3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:13,422 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:13,422 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/B, priority=13, startTime=1733193373382; duration=0sec 2024-12-03T02:36:13,422 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:13,422 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:B 2024-12-03T02:36:13,423 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:13,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742450_1626 (size=4469) 2024-12-03T02:36:13,429 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:13,429 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/C is initiating minor compaction (all files) 2024-12-03T02:36:13,429 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/C in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:13,429 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/76b29038d3884e729fbd74997c161d8f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ae33f2d0fe5241728f9598b418cd8784, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3b4a6d6b69547d196263a724db29d4f] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.6 K 2024-12-03T02:36:13,429 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 76b29038d3884e729fbd74997c161d8f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733193370847 2024-12-03T02:36:13,429 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ae33f2d0fe5241728f9598b418cd8784, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733193371183 2024-12-03T02:36:13,430 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b4a6d6b69547d196263a724db29d4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193372568 2024-12-03T02:36:13,438 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#C#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:13,438 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/14fc35b74e1840a1a9a1c33505d5be36 is 50, key is test_row_0/C:col10/1733193372568/Put/seqid=0 2024-12-03T02:36:13,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742451_1627 (size=13017) 2024-12-03T02:36:13,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T02:36:13,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:13,448 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T02:36:13,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:13,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:13,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:13,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:13,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:13,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:13,457 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/14fc35b74e1840a1a9a1c33505d5be36 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/14fc35b74e1840a1a9a1c33505d5be36 2024-12-03T02:36:13,461 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/C of 0be0304c4b25813eaf7af443d2898000 into 14fc35b74e1840a1a9a1c33505d5be36(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:13,461 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:13,461 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/C, priority=13, startTime=1733193373382; duration=0sec 2024-12-03T02:36:13,461 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:13,461 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:C 2024-12-03T02:36:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a86d9af9415a473a951763a512b78e56_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193372627/Put/seqid=0 2024-12-03T02:36:13,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742452_1628 (size=12454) 2024-12-03T02:36:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T02:36:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:13,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193433798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193433799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193433799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:13,829 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#A#compaction#520 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:13,829 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/174a6558523c45c3800b61d9ef651bd3 is 175, key is test_row_0/A:col10/1733193372568/Put/seqid=0 2024-12-03T02:36:13,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742453_1629 (size=31971) 2024-12-03T02:36:13,835 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/174a6558523c45c3800b61d9ef651bd3 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/174a6558523c45c3800b61d9ef651bd3 2024-12-03T02:36:13,839 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/A of 0be0304c4b25813eaf7af443d2898000 into 174a6558523c45c3800b61d9ef651bd3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:13,839 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:13,839 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/A, priority=13, startTime=1733193373382; duration=0sec 2024-12-03T02:36:13,839 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:13,839 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:A 2024-12-03T02:36:13,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:13,872 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a86d9af9415a473a951763a512b78e56_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a86d9af9415a473a951763a512b78e56_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:13,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/dc0a324315d647e4b3ca9a0d787ce44c, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:13,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/dc0a324315d647e4b3ca9a0d787ce44c is 175, key is test_row_0/A:col10/1733193372627/Put/seqid=0 2024-12-03T02:36:13,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742454_1630 (size=31255) 2024-12-03T02:36:13,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193433905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:13,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193433905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:13,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193433905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193434115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193434115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193434115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,281 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/dc0a324315d647e4b3ca9a0d787ce44c 2024-12-03T02:36:14,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/3c4b40b69fc5462f9eada6933e9d8db4 is 50, key is test_row_0/B:col10/1733193372627/Put/seqid=0 2024-12-03T02:36:14,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742455_1631 (size=12301) 2024-12-03T02:36:14,305 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/3c4b40b69fc5462f9eada6933e9d8db4 2024-12-03T02:36:14,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/9f7822645ba74505aa46652be90c2a00 is 50, key is test_row_0/C:col10/1733193372627/Put/seqid=0 2024-12-03T02:36:14,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742456_1632 (size=12301) 2024-12-03T02:36:14,319 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/9f7822645ba74505aa46652be90c2a00 2024-12-03T02:36:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/dc0a324315d647e4b3ca9a0d787ce44c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dc0a324315d647e4b3ca9a0d787ce44c 2024-12-03T02:36:14,328 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dc0a324315d647e4b3ca9a0d787ce44c, entries=150, sequenceid=359, filesize=30.5 K 2024-12-03T02:36:14,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/3c4b40b69fc5462f9eada6933e9d8db4 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/3c4b40b69fc5462f9eada6933e9d8db4 2024-12-03T02:36:14,332 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/3c4b40b69fc5462f9eada6933e9d8db4, entries=150, sequenceid=359, filesize=12.0 K 2024-12-03T02:36:14,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/9f7822645ba74505aa46652be90c2a00 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f7822645ba74505aa46652be90c2a00 2024-12-03T02:36:14,339 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f7822645ba74505aa46652be90c2a00, entries=150, sequenceid=359, filesize=12.0 K 2024-12-03T02:36:14,341 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 0be0304c4b25813eaf7af443d2898000 in 894ms, sequenceid=359, compaction requested=false 2024-12-03T02:36:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-03T02:36:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-03T02:36:14,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-03T02:36:14,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-12-03T02:36:14,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.8310 sec 2024-12-03T02:36:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:14,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T02:36:14,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:14,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:14,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:14,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:14,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:14,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:14,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d7a9f4ffde104671a2e035ecc6f19721_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193374426/Put/seqid=0 2024-12-03T02:36:14,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742457_1633 (size=14994) 2024-12-03T02:36:14,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193434461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193434463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193434463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193434575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193434575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193434575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T02:36:14,675 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:14,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T02:36:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-03T02:36:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T02:36:14,678 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T02:36:14,680 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T02:36:14,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T02:36:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T02:36:14,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193434787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193434787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193434787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:14,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:14,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:14,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:14,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:14,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:14,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:14,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:14,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:14,841 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d7a9f4ffde104671a2e035ecc6f19721_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d7a9f4ffde104671a2e035ecc6f19721_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:14,846 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/7ad1e09873154c7bbbcfd4ec832775b9, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:14,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/7ad1e09873154c7bbbcfd4ec832775b9 is 175, key is test_row_0/A:col10/1733193374426/Put/seqid=0 2024-12-03T02:36:14,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742458_1634 (size=39949) 2024-12-03T02:36:14,887 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/7ad1e09873154c7bbbcfd4ec832775b9 2024-12-03T02:36:14,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/40baeb3871114625898dfdbbc3c53fd1 is 50, key is test_row_0/B:col10/1733193374426/Put/seqid=0 2024-12-03T02:36:14,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742459_1635 (size=12301) 2024-12-03T02:36:14,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/40baeb3871114625898dfdbbc3c53fd1 2024-12-03T02:36:14,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/998eb948c087492781097320da787891 is 50, key is test_row_0/C:col10/1733193374426/Put/seqid=0 2024-12-03T02:36:14,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:14,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:14,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:14,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:14,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:14,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T02:36:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742460_1636 (size=12301) 2024-12-03T02:36:15,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193435105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193435105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193435108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:15,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T02:36:15,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/998eb948c087492781097320da787891 2024-12-03T02:36:15,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:15,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:15,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/7ad1e09873154c7bbbcfd4ec832775b9 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7ad1e09873154c7bbbcfd4ec832775b9 2024-12-03T02:36:15,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7ad1e09873154c7bbbcfd4ec832775b9, entries=200, sequenceid=381, filesize=39.0 K 2024-12-03T02:36:15,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/40baeb3871114625898dfdbbc3c53fd1 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/40baeb3871114625898dfdbbc3c53fd1 2024-12-03T02:36:15,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/40baeb3871114625898dfdbbc3c53fd1, entries=150, sequenceid=381, filesize=12.0 K 2024-12-03T02:36:15,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/998eb948c087492781097320da787891 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/998eb948c087492781097320da787891 2024-12-03T02:36:15,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/998eb948c087492781097320da787891, entries=150, sequenceid=381, filesize=12.0 K 2024-12-03T02:36:15,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0be0304c4b25813eaf7af443d2898000 in 1139ms, sequenceid=381, compaction requested=true 2024-12-03T02:36:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:36:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:15,566 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:36:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:36:15,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:36:15,566 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:15,567 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:15,567 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/B is initiating minor compaction (all files) 2024-12-03T02:36:15,567 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/B in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,567 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d4660dfc4e3648eab86728f09ac3d5e3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/3c4b40b69fc5462f9eada6933e9d8db4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/40baeb3871114625898dfdbbc3c53fd1] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.7 K 2024-12-03T02:36:15,568 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:15,568 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/A is initiating minor compaction (all files) 2024-12-03T02:36:15,568 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/A in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,568 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/174a6558523c45c3800b61d9ef651bd3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dc0a324315d647e4b3ca9a0d787ce44c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7ad1e09873154c7bbbcfd4ec832775b9] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=100.8 K 2024-12-03T02:36:15,568 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,568 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/174a6558523c45c3800b61d9ef651bd3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dc0a324315d647e4b3ca9a0d787ce44c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7ad1e09873154c7bbbcfd4ec832775b9] 2024-12-03T02:36:15,568 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting d4660dfc4e3648eab86728f09ac3d5e3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193372568 2024-12-03T02:36:15,568 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 174a6558523c45c3800b61d9ef651bd3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193372568 2024-12-03T02:36:15,568 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c4b40b69fc5462f9eada6933e9d8db4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733193372597 2024-12-03T02:36:15,568 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc0a324315d647e4b3ca9a0d787ce44c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733193372597 2024-12-03T02:36:15,569 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 40baeb3871114625898dfdbbc3c53fd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733193373790 2024-12-03T02:36:15,569 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad1e09873154c7bbbcfd4ec832775b9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733193373790 2024-12-03T02:36:15,595 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:15,602 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#B#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:15,602 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/2ee0a871ccb34519a3043b19077e38fa is 50, key is test_row_0/B:col10/1733193374426/Put/seqid=0 2024-12-03T02:36:15,613 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203cc2134611a6e4d27810f7f56159c8fda_0be0304c4b25813eaf7af443d2898000 store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:15,615 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203cc2134611a6e4d27810f7f56159c8fda_0be0304c4b25813eaf7af443d2898000, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:15,615 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cc2134611a6e4d27810f7f56159c8fda_0be0304c4b25813eaf7af443d2898000 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:15,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T02:36:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:15,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:15,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203063d4fbd8b844268a842fc41d15920b2_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193375620/Put/seqid=0 2024-12-03T02:36:15,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742461_1637 (size=13119) 2024-12-03T02:36:15,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742462_1638 (size=4469) 2024-12-03T02:36:15,661 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#A#compaction#529 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:15,662 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/1067c1162f024a359ca7632c877d9983 is 175, key is test_row_0/A:col10/1733193374426/Put/seqid=0 2024-12-03T02:36:15,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193435676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193435678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:15,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193435685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:15,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742464_1640 (size=32073) 2024-12-03T02:36:15,709 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/1067c1162f024a359ca7632c877d9983 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/1067c1162f024a359ca7632c877d9983 2024-12-03T02:36:15,713 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/A of 0be0304c4b25813eaf7af443d2898000 into 1067c1162f024a359ca7632c877d9983(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:15,713 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:15,713 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/A, priority=13, startTime=1733193375566; duration=0sec 2024-12-03T02:36:15,713 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:15,713 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:A 2024-12-03T02:36:15,713 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:15,714 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:15,715 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/C is initiating minor compaction (all files) 2024-12-03T02:36:15,715 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/C in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,715 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/14fc35b74e1840a1a9a1c33505d5be36, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f7822645ba74505aa46652be90c2a00, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/998eb948c087492781097320da787891] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.7 K 2024-12-03T02:36:15,715 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14fc35b74e1840a1a9a1c33505d5be36, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733193372568 2024-12-03T02:36:15,715 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f7822645ba74505aa46652be90c2a00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733193372597 2024-12-03T02:36:15,716 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 998eb948c087492781097320da787891, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733193373790 2024-12-03T02:36:15,737 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#C#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:15,737 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/b316d88586844a26942ee768e3c92eff is 50, key is test_row_0/C:col10/1733193374426/Put/seqid=0 2024-12-03T02:36:15,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742463_1639 (size=14994) 2024-12-03T02:36:15,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742465_1641 (size=13119) 2024-12-03T02:36:15,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193435795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:15,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:15,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193435795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193435795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:15,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T02:36:15,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:15,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:15,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:15,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193436004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193436004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193436004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,063 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/2ee0a871ccb34519a3043b19077e38fa as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2ee0a871ccb34519a3043b19077e38fa 2024-12-03T02:36:16,067 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/B of 0be0304c4b25813eaf7af443d2898000 into 2ee0a871ccb34519a3043b19077e38fa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:16,067 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:16,067 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/B, priority=13, startTime=1733193375566; duration=0sec 2024-12-03T02:36:16,067 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:16,067 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:B 2024-12-03T02:36:16,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,144 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:16,159 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203063d4fbd8b844268a842fc41d15920b2_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203063d4fbd8b844268a842fc41d15920b2_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:16,160 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/5d79ba45c5194f73aae369bf5ee7f782, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:16,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/5d79ba45c5194f73aae369bf5ee7f782 is 175, key is test_row_0/A:col10/1733193375620/Put/seqid=0 2024-12-03T02:36:16,179 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/b316d88586844a26942ee768e3c92eff as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/b316d88586844a26942ee768e3c92eff 2024-12-03T02:36:16,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742466_1642 (size=39949) 2024-12-03T02:36:16,189 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/5d79ba45c5194f73aae369bf5ee7f782 2024-12-03T02:36:16,191 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/C of 0be0304c4b25813eaf7af443d2898000 into b316d88586844a26942ee768e3c92eff(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:16,192 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:16,192 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/C, priority=13, startTime=1733193375566; duration=0sec 2024-12-03T02:36:16,192 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:16,192 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:C 2024-12-03T02:36:16,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/1d9d4aed1cab4c06bfa25cb36dfd352b is 50, key is test_row_0/B:col10/1733193375620/Put/seqid=0 2024-12-03T02:36:16,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742467_1643 (size=12301) 2024-12-03T02:36:16,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:16,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:16,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193436315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193436319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193436319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:16,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:16,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/1d9d4aed1cab4c06bfa25cb36dfd352b 2024-12-03T02:36:16,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/37f96729af824b9da65f8bafe1738b33 is 50, key is test_row_0/C:col10/1733193375620/Put/seqid=0 2024-12-03T02:36:16,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742468_1644 (size=12301) 2024-12-03T02:36:16,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/37f96729af824b9da65f8bafe1738b33 2024-12-03T02:36:16,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/5d79ba45c5194f73aae369bf5ee7f782 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/5d79ba45c5194f73aae369bf5ee7f782 2024-12-03T02:36:16,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/5d79ba45c5194f73aae369bf5ee7f782, entries=200, sequenceid=395, filesize=39.0 K 2024-12-03T02:36:16,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/1d9d4aed1cab4c06bfa25cb36dfd352b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/1d9d4aed1cab4c06bfa25cb36dfd352b 2024-12-03T02:36:16,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36054 deadline: 1733193436734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,738 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 64035cbb0b46,38721,1733193202350 for 'test_row_2' in 0be0304c4b25813eaf7af443d2898000 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:36:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/1d9d4aed1cab4c06bfa25cb36dfd352b, entries=150, sequenceid=395, filesize=12.0 K 2024-12-03T02:36:16,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/37f96729af824b9da65f8bafe1738b33 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37f96729af824b9da65f8bafe1738b33 2024-12-03T02:36:16,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37f96729af824b9da65f8bafe1738b33, entries=150, sequenceid=395, filesize=12.0 K 2024-12-03T02:36:16,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0be0304c4b25813eaf7af443d2898000 in 1151ms, sequenceid=395, compaction requested=false 2024-12-03T02:36:16,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:16,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T02:36:16,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:16,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:16,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:16,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:16,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:16,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T02:36:16,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203662d0a90f6e34e3ab394f3b411762e19_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193376795/Put/seqid=0 2024-12-03T02:36:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193436846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193436849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193436849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193436851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742469_1645 (size=12454) 2024-12-03T02:36:16,896 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:16,905 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203662d0a90f6e34e3ab394f3b411762e19_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203662d0a90f6e34e3ab394f3b411762e19_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:16,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/24997f7e50dd460e9919d2e8fed2b160, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:16,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/24997f7e50dd460e9919d2e8fed2b160 is 175, key is test_row_0/A:col10/1733193376795/Put/seqid=0 2024-12-03T02:36:16,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:16,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:16,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:16,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:16,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742470_1646 (size=31255) 2024-12-03T02:36:16,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193436955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193436966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193436967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:16,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:17,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193437164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:17,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193437174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193437187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:17,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,333 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=417, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/24997f7e50dd460e9919d2e8fed2b160 2024-12-03T02:36:17,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/b8d4553c36a34f69a057930e486ff982 is 50, key is test_row_0/B:col10/1733193376795/Put/seqid=0 2024-12-03T02:36:17,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742471_1647 (size=12301) 2024-12-03T02:36:17,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:17,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193437475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:17,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193437484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:17,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193437494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:17,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:17,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:17,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:36:17,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/b8d4553c36a34f69a057930e486ff982 2024-12-03T02:36:17,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/f78005aa231f4f21a4129b732c86d63c is 50, key is test_row_0/C:col10/1733193376795/Put/seqid=0 2024-12-03T02:36:17,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742472_1648 (size=12301) 2024-12-03T02:36:17,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/f78005aa231f4f21a4129b732c86d63c 2024-12-03T02:36:17,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/24997f7e50dd460e9919d2e8fed2b160 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/24997f7e50dd460e9919d2e8fed2b160 2024-12-03T02:36:17,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/24997f7e50dd460e9919d2e8fed2b160, entries=150, sequenceid=417, filesize=30.5 K 2024-12-03T02:36:17,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/b8d4553c36a34f69a057930e486ff982 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b8d4553c36a34f69a057930e486ff982 2024-12-03T02:36:17,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b8d4553c36a34f69a057930e486ff982, entries=150, sequenceid=417, filesize=12.0 K 2024-12-03T02:36:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/f78005aa231f4f21a4129b732c86d63c as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/f78005aa231f4f21a4129b732c86d63c 2024-12-03T02:36:17,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/f78005aa231f4f21a4129b732c86d63c, entries=150, sequenceid=417, filesize=12.0 K 2024-12-03T02:36:17,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0be0304c4b25813eaf7af443d2898000 in 1009ms, sequenceid=417, compaction requested=true 2024-12-03T02:36:17,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:17,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:36:17,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:17,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:36:17,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T02:36:17,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:36:17,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T02:36:17,815 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:17,815 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:17,824 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:17,824 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/A is initiating minor compaction (all files) 2024-12-03T02:36:17,825 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/A in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,825 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/1067c1162f024a359ca7632c877d9983, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/5d79ba45c5194f73aae369bf5ee7f782, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/24997f7e50dd460e9919d2e8fed2b160] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=100.9 K 2024-12-03T02:36:17,825 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,825 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/1067c1162f024a359ca7632c877d9983, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/5d79ba45c5194f73aae369bf5ee7f782, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/24997f7e50dd460e9919d2e8fed2b160] 2024-12-03T02:36:17,827 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:17,827 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/C is initiating minor compaction (all files) 2024-12-03T02:36:17,827 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/C in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,827 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/b316d88586844a26942ee768e3c92eff, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37f96729af824b9da65f8bafe1738b33, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/f78005aa231f4f21a4129b732c86d63c] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.8 K 2024-12-03T02:36:17,829 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b316d88586844a26942ee768e3c92eff, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733193373790 2024-12-03T02:36:17,830 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 37f96729af824b9da65f8bafe1738b33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733193374461 2024-12-03T02:36:17,831 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1067c1162f024a359ca7632c877d9983, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733193373790 2024-12-03T02:36:17,835 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d79ba45c5194f73aae369bf5ee7f782, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733193374461 2024-12-03T02:36:17,835 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting f78005aa231f4f21a4129b732c86d63c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733193375651 2024-12-03T02:36:17,836 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24997f7e50dd460e9919d2e8fed2b160, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733193375651 2024-12-03T02:36:17,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38721 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T02:36:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,855 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T02:36:17,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:17,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:17,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:17,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:17,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:17,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:17,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. as already flushing 2024-12-03T02:36:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:17,877 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:17,894 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#C#compaction#539 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:17,895 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/663dd2e3d1c04d76b51d19544f2d652d is 50, key is test_row_0/C:col10/1733193376795/Put/seqid=0 2024-12-03T02:36:17,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f05e4a72040a4c1080503440b77fe034_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193376841/Put/seqid=0 2024-12-03T02:36:17,904 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120345232d8ff41f444fb3a04a47958f5f08_0be0304c4b25813eaf7af443d2898000 store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:17,906 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120345232d8ff41f444fb3a04a47958f5f08_0be0304c4b25813eaf7af443d2898000, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:17,906 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120345232d8ff41f444fb3a04a47958f5f08_0be0304c4b25813eaf7af443d2898000 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:17,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742473_1649 (size=13221) 2024-12-03T02:36:17,942 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/663dd2e3d1c04d76b51d19544f2d652d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/663dd2e3d1c04d76b51d19544f2d652d 2024-12-03T02:36:17,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742474_1650 (size=12454) 2024-12-03T02:36:17,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:17,952 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/C of 0be0304c4b25813eaf7af443d2898000 into 663dd2e3d1c04d76b51d19544f2d652d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:17,952 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:17,952 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/C, priority=13, startTime=1733193377813; duration=0sec 2024-12-03T02:36:17,952 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:17,952 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:C 2024-12-03T02:36:17,952 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:17,953 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f05e4a72040a4c1080503440b77fe034_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f05e4a72040a4c1080503440b77fe034_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:17,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/740e1d03cb584b39ab32cadc0635003a, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:17,955 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/740e1d03cb584b39ab32cadc0635003a is 175, key is test_row_0/A:col10/1733193376841/Put/seqid=0 2024-12-03T02:36:17,955 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/B is initiating minor compaction (all files) 2024-12-03T02:36:17,955 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/B in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:17,955 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2ee0a871ccb34519a3043b19077e38fa, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/1d9d4aed1cab4c06bfa25cb36dfd352b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b8d4553c36a34f69a057930e486ff982] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.8 K 2024-12-03T02:36:17,955 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ee0a871ccb34519a3043b19077e38fa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1733193373790 2024-12-03T02:36:17,956 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d9d4aed1cab4c06bfa25cb36dfd352b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733193374461 2024-12-03T02:36:17,956 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting b8d4553c36a34f69a057930e486ff982, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733193375651 2024-12-03T02:36:17,972 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#B#compaction#541 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:17,973 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/289897cf415a48ea8b50ebdf576a4f99 is 50, key is test_row_0/B:col10/1733193376795/Put/seqid=0 2024-12-03T02:36:17,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742476_1652 (size=31255) 2024-12-03T02:36:17,975 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=430, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/740e1d03cb584b39ab32cadc0635003a 2024-12-03T02:36:17,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742475_1651 (size=4469) 2024-12-03T02:36:17,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742477_1653 (size=13221) 2024-12-03T02:36:17,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/4f17fe443c174ffb9af2816f64b51178 is 50, key is test_row_0/B:col10/1733193376841/Put/seqid=0 2024-12-03T02:36:17,990 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#A#compaction#538 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:17,990 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/052e0dfa3cf240b48fe30c02849a49df is 175, key is test_row_0/A:col10/1733193376795/Put/seqid=0 2024-12-03T02:36:17,995 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/289897cf415a48ea8b50ebdf576a4f99 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/289897cf415a48ea8b50ebdf576a4f99 2024-12-03T02:36:17,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36092 deadline: 1733193437994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:18,001 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/B of 0be0304c4b25813eaf7af443d2898000 into 289897cf415a48ea8b50ebdf576a4f99(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:18,001 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:18,001 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/B, priority=13, startTime=1733193377813; duration=0sec 2024-12-03T02:36:18,001 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:18,001 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:B 2024-12-03T02:36:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193437998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193437998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:18,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:18,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36056 deadline: 1733193438004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:18,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742479_1655 (size=32175) 2024-12-03T02:36:18,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742478_1654 (size=12301) 2024-12-03T02:36:18,016 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/4f17fe443c174ffb9af2816f64b51178 2024-12-03T02:36:18,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/20bee25e2ec745e3a773c8e86cf729e0 is 50, key is test_row_0/C:col10/1733193376841/Put/seqid=0 2024-12-03T02:36:18,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742480_1656 (size=12301) 2024-12-03T02:36:18,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193438114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193438114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:18,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:18,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:36:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36046 deadline: 1733193438325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38721 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36032 deadline: 1733193438325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 , the old value is region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:36:18,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., hostname=64035cbb0b46,38721,1733193202350, seqNum=5 because the exception is null or not the one we care about 2024-12-03T02:36:18,419 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/052e0dfa3cf240b48fe30c02849a49df as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/052e0dfa3cf240b48fe30c02849a49df 2024-12-03T02:36:18,422 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/A of 0be0304c4b25813eaf7af443d2898000 into 052e0dfa3cf240b48fe30c02849a49df(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:18,422 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:18,422 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/A, priority=13, startTime=1733193377813; duration=0sec 2024-12-03T02:36:18,422 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:18,422 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:A 2024-12-03T02:36:18,423 INFO [Thread-2419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2419. 2024-12-03T02:36:18,423 INFO [Thread-2421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2421. 2024-12-03T02:36:18,424 DEBUG [Thread-2419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:18,424 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,424 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:18,424 DEBUG [Thread-2421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:18,424 DEBUG [Thread-2421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,424 DEBUG [Thread-2421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,424 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:18,424 DEBUG [Thread-2419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,428 INFO [Thread-2417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2417. 2024-12-03T02:36:18,428 DEBUG [Thread-2417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:18,428 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,428 DEBUG [Thread-2417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:18,428 INFO [Thread-2423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2423. 2024-12-03T02:36:18,428 DEBUG [Thread-2423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:18,428 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,428 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:18,429 INFO [Thread-2415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2415. 2024-12-03T02:36:18,429 DEBUG [Thread-2415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:18,429 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:18,429 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,437 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/20bee25e2ec745e3a773c8e86cf729e0 2024-12-03T02:36:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/740e1d03cb584b39ab32cadc0635003a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/740e1d03cb584b39ab32cadc0635003a 2024-12-03T02:36:18,445 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/740e1d03cb584b39ab32cadc0635003a, entries=150, sequenceid=430, filesize=30.5 K 2024-12-03T02:36:18,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/4f17fe443c174ffb9af2816f64b51178 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/4f17fe443c174ffb9af2816f64b51178 2024-12-03T02:36:18,449 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/4f17fe443c174ffb9af2816f64b51178, entries=150, sequenceid=430, filesize=12.0 K 2024-12-03T02:36:18,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/20bee25e2ec745e3a773c8e86cf729e0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/20bee25e2ec745e3a773c8e86cf729e0 2024-12-03T02:36:18,453 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/20bee25e2ec745e3a773c8e86cf729e0, entries=150, sequenceid=430, filesize=12.0 K 2024-12-03T02:36:18,453 INFO [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0be0304c4b25813eaf7af443d2898000 in 598ms, sequenceid=430, compaction requested=false 2024-12-03T02:36:18,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:18,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:18,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-03T02:36:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-03T02:36:18,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-03T02:36:18,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7750 sec 2024-12-03T02:36:18,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 3.7820 sec 2024-12-03T02:36:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38721 {}] regionserver.HRegion(8855): Flush requested on 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:18,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T02:36:18,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:18,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:18,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:18,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:18,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:18,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T02:36:18,637 INFO [Thread-2410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2410. 2024-12-03T02:36:18,638 DEBUG [Thread-2410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:18,638 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,638 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:18,638 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-12-03T02:36:18,638 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:18,638 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,638 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:18,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:18,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cde2adb852f4678885b3188b044bf33_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_0/A:col10/1733193377995/Put/seqid=0 2024-12-03T02:36:18,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742481_1657 (size=12454) 2024-12-03T02:36:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T02:36:18,855 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:19,006 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-12-03T02:36:19,007 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:19,007 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:19,007 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:19,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:19,016 INFO [Thread-2412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2412. 2024-12-03T02:36:19,016 DEBUG [Thread-2412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:19,016 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:19,016 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:19,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:19,048 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:19,052 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cde2adb852f4678885b3188b044bf33_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cde2adb852f4678885b3188b044bf33_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:19,052 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ba27fda479e446a692684bc04aca409e, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:19,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ba27fda479e446a692684bc04aca409e is 175, key is test_row_0/A:col10/1733193377995/Put/seqid=0 2024-12-03T02:36:19,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742482_1658 (size=31255) 2024-12-03T02:36:19,457 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=454, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ba27fda479e446a692684bc04aca409e 2024-12-03T02:36:19,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/730052e4c83a461bbb01084a656e3c60 is 50, key is test_row_0/B:col10/1733193377995/Put/seqid=0 2024-12-03T02:36:19,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742483_1659 (size=12301) 2024-12-03T02:36:19,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/730052e4c83a461bbb01084a656e3c60 2024-12-03T02:36:19,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/e7ec4836acd34fbcbf9a2e6db865fdd2 is 50, key is test_row_0/C:col10/1733193377995/Put/seqid=0 2024-12-03T02:36:19,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742484_1660 (size=12301) 2024-12-03T02:36:20,004 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T02:36:20,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/e7ec4836acd34fbcbf9a2e6db865fdd2 2024-12-03T02:36:20,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ba27fda479e446a692684bc04aca409e as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ba27fda479e446a692684bc04aca409e 2024-12-03T02:36:20,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ba27fda479e446a692684bc04aca409e, entries=150, sequenceid=454, filesize=30.5 K 2024-12-03T02:36:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/730052e4c83a461bbb01084a656e3c60 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/730052e4c83a461bbb01084a656e3c60 2024-12-03T02:36:20,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/730052e4c83a461bbb01084a656e3c60, entries=150, sequenceid=454, filesize=12.0 K 2024-12-03T02:36:20,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/e7ec4836acd34fbcbf9a2e6db865fdd2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/e7ec4836acd34fbcbf9a2e6db865fdd2 2024-12-03T02:36:20,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/e7ec4836acd34fbcbf9a2e6db865fdd2, entries=150, sequenceid=454, filesize=12.0 K 2024-12-03T02:36:20,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=13.42 KB/13740 for 0be0304c4b25813eaf7af443d2898000 in 1659ms, sequenceid=454, compaction requested=true 2024-12-03T02:36:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T02:36:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T02:36:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:20,295 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0be0304c4b25813eaf7af443d2898000:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T02:36:20,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:20,295 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:20,296 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:20,296 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:20,296 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/B is initiating minor compaction (all files) 2024-12-03T02:36:20,296 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/A is initiating minor compaction (all files) 2024-12-03T02:36:20,296 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/A in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:20,296 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/B in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:20,296 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/289897cf415a48ea8b50ebdf576a4f99, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/4f17fe443c174ffb9af2816f64b51178, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/730052e4c83a461bbb01084a656e3c60] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.9 K 2024-12-03T02:36:20,296 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/052e0dfa3cf240b48fe30c02849a49df, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/740e1d03cb584b39ab32cadc0635003a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ba27fda479e446a692684bc04aca409e] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=92.5 K 2024-12-03T02:36:20,296 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:20,296 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. files: [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/052e0dfa3cf240b48fe30c02849a49df, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/740e1d03cb584b39ab32cadc0635003a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ba27fda479e446a692684bc04aca409e] 2024-12-03T02:36:20,296 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 289897cf415a48ea8b50ebdf576a4f99, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733193375651 2024-12-03T02:36:20,296 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 052e0dfa3cf240b48fe30c02849a49df, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733193375651 2024-12-03T02:36:20,296 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f17fe443c174ffb9af2816f64b51178, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733193376841 2024-12-03T02:36:20,297 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 740e1d03cb584b39ab32cadc0635003a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733193376841 2024-12-03T02:36:20,297 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] compactions.Compactor(225): Compacting 730052e4c83a461bbb01084a656e3c60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733193377995 2024-12-03T02:36:20,297 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting ba27fda479e446a692684bc04aca409e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733193377995 2024-12-03T02:36:20,302 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#B#compaction#547 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:20,302 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/8935e51dd8a9491895dcd8f112b2fd20 is 50, key is test_row_0/B:col10/1733193377995/Put/seqid=0 2024-12-03T02:36:20,303 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:20,304 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203460a823291844c03b1f22fab84a23589_0be0304c4b25813eaf7af443d2898000 store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:20,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742485_1661 (size=13323) 2024-12-03T02:36:20,310 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203460a823291844c03b1f22fab84a23589_0be0304c4b25813eaf7af443d2898000, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:20,310 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203460a823291844c03b1f22fab84a23589_0be0304c4b25813eaf7af443d2898000 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:20,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742486_1662 (size=4469) 2024-12-03T02:36:20,320 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#A#compaction#548 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:20,321 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/3431815651f64edeb510cf00b6b23484 is 175, key is test_row_0/A:col10/1733193377995/Put/seqid=0 2024-12-03T02:36:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742487_1663 (size=32277) 2024-12-03T02:36:20,357 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/3431815651f64edeb510cf00b6b23484 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3431815651f64edeb510cf00b6b23484 2024-12-03T02:36:20,366 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/A of 0be0304c4b25813eaf7af443d2898000 into 3431815651f64edeb510cf00b6b23484(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:20,366 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:20,366 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/A, priority=13, startTime=1733193380295; duration=0sec 2024-12-03T02:36:20,366 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T02:36:20,366 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:A 2024-12-03T02:36:20,366 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T02:36:20,367 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T02:36:20,367 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1541): 0be0304c4b25813eaf7af443d2898000/C is initiating minor compaction (all files) 2024-12-03T02:36:20,367 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0be0304c4b25813eaf7af443d2898000/C in TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:20,367 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/663dd2e3d1c04d76b51d19544f2d652d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/20bee25e2ec745e3a773c8e86cf729e0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/e7ec4836acd34fbcbf9a2e6db865fdd2] into tmpdir=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp, totalSize=36.9 K 2024-12-03T02:36:20,367 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 663dd2e3d1c04d76b51d19544f2d652d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733193375651 2024-12-03T02:36:20,367 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting 20bee25e2ec745e3a773c8e86cf729e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733193376841 2024-12-03T02:36:20,368 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] compactions.Compactor(225): Compacting e7ec4836acd34fbcbf9a2e6db865fdd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733193377995 2024-12-03T02:36:20,373 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0be0304c4b25813eaf7af443d2898000#C#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T02:36:20,373 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/8d9e22b7c04e4e64ad7a88aab3d7b3d0 is 50, key is test_row_0/C:col10/1733193377995/Put/seqid=0 2024-12-03T02:36:20,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742488_1664 (size=13323) 2024-12-03T02:36:20,709 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/8935e51dd8a9491895dcd8f112b2fd20 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8935e51dd8a9491895dcd8f112b2fd20 2024-12-03T02:36:20,713 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/B of 0be0304c4b25813eaf7af443d2898000 into 8935e51dd8a9491895dcd8f112b2fd20(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:20,713 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:20,713 INFO [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/B, priority=13, startTime=1733193380295; duration=0sec 2024-12-03T02:36:20,713 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:20,713 DEBUG [RS:0;64035cbb0b46:38721-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:B 2024-12-03T02:36:20,780 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/8d9e22b7c04e4e64ad7a88aab3d7b3d0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/8d9e22b7c04e4e64ad7a88aab3d7b3d0 2024-12-03T02:36:20,784 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0be0304c4b25813eaf7af443d2898000/C of 0be0304c4b25813eaf7af443d2898000 into 8d9e22b7c04e4e64ad7a88aab3d7b3d0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T02:36:20,784 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0be0304c4b25813eaf7af443d2898000: 2024-12-03T02:36:20,784 INFO [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., storeName=0be0304c4b25813eaf7af443d2898000/C, priority=13, startTime=1733193380295; duration=0sec 2024-12-03T02:36:20,784 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T02:36:20,784 DEBUG [RS:0;64035cbb0b46:38721-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0be0304c4b25813eaf7af443d2898000:C 2024-12-03T02:36:26,776 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-12-03T02:36:26,776 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T02:36:26,776 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:26,776 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:26,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3117 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2999 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2978 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3118 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2986 2024-12-03T02:36:26,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T02:36:26,777 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:36:26,777 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:36:26,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:26,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:26,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:26,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T02:36:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T02:36:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T02:36:26,780 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193386780"}]},"ts":"1733193386780"} 2024-12-03T02:36:26,781 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T02:36:26,781 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T02:36:26,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T02:36:26,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, UNASSIGN}] 2024-12-03T02:36:26,784 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, UNASSIGN 2024-12-03T02:36:26,784 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=0be0304c4b25813eaf7af443d2898000, regionState=CLOSING, regionLocation=64035cbb0b46,38721,1733193202350 2024-12-03T02:36:26,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, UNASSIGN because future has completed 2024-12-03T02:36:26,786 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T02:36:26,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350}] 2024-12-03T02:36:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T02:36:26,938 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 0be0304c4b25813eaf7af443d2898000, disabling compactions & flushes 2024-12-03T02:36:26,938 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. after waiting 0 ms 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:26,938 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 0be0304c4b25813eaf7af443d2898000 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=A 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=B 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0be0304c4b25813eaf7af443d2898000, store=C 2024-12-03T02:36:26,938 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T02:36:26,943 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba657e4857b7400588280ffb210588df_0be0304c4b25813eaf7af443d2898000 is 50, key is test_row_1/A:col10/1733193379015/Put/seqid=0 2024-12-03T02:36:26,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742489_1665 (size=9914) 2024-12-03T02:36:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T02:36:27,346 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:36:27,350 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba657e4857b7400588280ffb210588df_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba657e4857b7400588280ffb210588df_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:27,350 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ef4c24c7ba7047909b79c53fa8f4a0ee, store: [table=TestAcidGuarantees family=A region=0be0304c4b25813eaf7af443d2898000] 2024-12-03T02:36:27,351 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ef4c24c7ba7047909b79c53fa8f4a0ee is 175, key is test_row_1/A:col10/1733193379015/Put/seqid=0 2024-12-03T02:36:27,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742490_1666 (size=22561) 2024-12-03T02:36:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T02:36:27,754 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ef4c24c7ba7047909b79c53fa8f4a0ee 2024-12-03T02:36:27,759 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/ce36c46d187a44039e2e1e497544bbc8 is 50, key is test_row_1/B:col10/1733193379015/Put/seqid=0 2024-12-03T02:36:27,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742491_1667 (size=9857) 2024-12-03T02:36:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T02:36:28,162 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/ce36c46d187a44039e2e1e497544bbc8 2024-12-03T02:36:28,169 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/b42bf22b1f36492dbfe7ff012519b40d is 50, key is test_row_1/C:col10/1733193379015/Put/seqid=0 2024-12-03T02:36:28,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742492_1668 (size=9857) 2024-12-03T02:36:28,172 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/b42bf22b1f36492dbfe7ff012519b40d 2024-12-03T02:36:28,178 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/A/ef4c24c7ba7047909b79c53fa8f4a0ee as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ef4c24c7ba7047909b79c53fa8f4a0ee 2024-12-03T02:36:28,182 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ef4c24c7ba7047909b79c53fa8f4a0ee, entries=100, sequenceid=463, filesize=22.0 K 2024-12-03T02:36:28,183 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/B/ce36c46d187a44039e2e1e497544bbc8 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/ce36c46d187a44039e2e1e497544bbc8 2024-12-03T02:36:28,187 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/ce36c46d187a44039e2e1e497544bbc8, entries=100, sequenceid=463, filesize=9.6 K 2024-12-03T02:36:28,187 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/.tmp/C/b42bf22b1f36492dbfe7ff012519b40d as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/b42bf22b1f36492dbfe7ff012519b40d 2024-12-03T02:36:28,191 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/b42bf22b1f36492dbfe7ff012519b40d, entries=100, sequenceid=463, filesize=9.6 K 2024-12-03T02:36:28,192 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0be0304c4b25813eaf7af443d2898000 in 1254ms, sequenceid=463, compaction requested=false 2024-12-03T02:36:28,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/270d396adf234ab4a22aafc698d7674d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/54a64c5cbabc4a6197216acbde78a534, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/00d95a638efd4c108bddba062b1ec740, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/9e8cc9e198044c9aac93366fa6613f72, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/bb2aebf3ad3a47009a33986476a3b0c5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/85cd720785b246d8af84133a642bcd9d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ab789ef9d7cf4b10ba5d1cf31c77cec7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/99f8d60c2cd94e739e7a2674afb76628, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/c64564070ad14c778037e311aa79a7f7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/aebd8d2f330e4a61ad08f9237026002d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ecc9fae55d434836bed22b91a0c38381, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dbd06c7af487458b855d381fee9a9d72, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3f75c853421e4e86b63b2b57feb9fc59, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/0a24b38d796d4ffea78fd4cd6b35c3e2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7068d999eee74c9dac880eddfc1ef96b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ea7f566b520c4d23a7d7aedcfee151b0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/eaa124b713644c4ea579606983e37155, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/f52c5803b0354a37968cadcc636c73b1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/abced3de92504d3f84a6d4b87c39cca1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/e1cfc3cfea154aca89313e3b35f2dca7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/174a6558523c45c3800b61d9ef651bd3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/8c9f4a7fe69a4f78b7f16f1339681ac9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dc0a324315d647e4b3ca9a0d787ce44c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7ad1e09873154c7bbbcfd4ec832775b9, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/1067c1162f024a359ca7632c877d9983, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/5d79ba45c5194f73aae369bf5ee7f782, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/052e0dfa3cf240b48fe30c02849a49df, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/24997f7e50dd460e9919d2e8fed2b160, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/740e1d03cb584b39ab32cadc0635003a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ba27fda479e446a692684bc04aca409e] to archive 2024-12-03T02:36:28,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:36:28,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/270d396adf234ab4a22aafc698d7674d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/270d396adf234ab4a22aafc698d7674d 2024-12-03T02:36:28,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/54a64c5cbabc4a6197216acbde78a534 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/54a64c5cbabc4a6197216acbde78a534 2024-12-03T02:36:28,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/00d95a638efd4c108bddba062b1ec740 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/00d95a638efd4c108bddba062b1ec740 2024-12-03T02:36:28,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/9e8cc9e198044c9aac93366fa6613f72 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/9e8cc9e198044c9aac93366fa6613f72 2024-12-03T02:36:28,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/bb2aebf3ad3a47009a33986476a3b0c5 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/bb2aebf3ad3a47009a33986476a3b0c5 2024-12-03T02:36:28,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/85cd720785b246d8af84133a642bcd9d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/85cd720785b246d8af84133a642bcd9d 2024-12-03T02:36:28,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ab789ef9d7cf4b10ba5d1cf31c77cec7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ab789ef9d7cf4b10ba5d1cf31c77cec7 2024-12-03T02:36:28,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/99f8d60c2cd94e739e7a2674afb76628 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/99f8d60c2cd94e739e7a2674afb76628 2024-12-03T02:36:28,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/c64564070ad14c778037e311aa79a7f7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/c64564070ad14c778037e311aa79a7f7 2024-12-03T02:36:28,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/aebd8d2f330e4a61ad08f9237026002d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/aebd8d2f330e4a61ad08f9237026002d 2024-12-03T02:36:28,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ecc9fae55d434836bed22b91a0c38381 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ecc9fae55d434836bed22b91a0c38381 2024-12-03T02:36:28,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dbd06c7af487458b855d381fee9a9d72 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dbd06c7af487458b855d381fee9a9d72 2024-12-03T02:36:28,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3f75c853421e4e86b63b2b57feb9fc59 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3f75c853421e4e86b63b2b57feb9fc59 2024-12-03T02:36:28,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/0a24b38d796d4ffea78fd4cd6b35c3e2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/0a24b38d796d4ffea78fd4cd6b35c3e2 2024-12-03T02:36:28,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7068d999eee74c9dac880eddfc1ef96b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7068d999eee74c9dac880eddfc1ef96b 2024-12-03T02:36:28,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ea7f566b520c4d23a7d7aedcfee151b0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ea7f566b520c4d23a7d7aedcfee151b0 2024-12-03T02:36:28,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/eaa124b713644c4ea579606983e37155 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/eaa124b713644c4ea579606983e37155 2024-12-03T02:36:28,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/f52c5803b0354a37968cadcc636c73b1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/f52c5803b0354a37968cadcc636c73b1 2024-12-03T02:36:28,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/abced3de92504d3f84a6d4b87c39cca1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/abced3de92504d3f84a6d4b87c39cca1 2024-12-03T02:36:28,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/e1cfc3cfea154aca89313e3b35f2dca7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/e1cfc3cfea154aca89313e3b35f2dca7 2024-12-03T02:36:28,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/174a6558523c45c3800b61d9ef651bd3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/174a6558523c45c3800b61d9ef651bd3 2024-12-03T02:36:28,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/8c9f4a7fe69a4f78b7f16f1339681ac9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/8c9f4a7fe69a4f78b7f16f1339681ac9 2024-12-03T02:36:28,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dc0a324315d647e4b3ca9a0d787ce44c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/dc0a324315d647e4b3ca9a0d787ce44c 2024-12-03T02:36:28,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7ad1e09873154c7bbbcfd4ec832775b9 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/7ad1e09873154c7bbbcfd4ec832775b9 2024-12-03T02:36:28,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/1067c1162f024a359ca7632c877d9983 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/1067c1162f024a359ca7632c877d9983 2024-12-03T02:36:28,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/5d79ba45c5194f73aae369bf5ee7f782 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/5d79ba45c5194f73aae369bf5ee7f782 2024-12-03T02:36:28,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/052e0dfa3cf240b48fe30c02849a49df to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/052e0dfa3cf240b48fe30c02849a49df 2024-12-03T02:36:28,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/24997f7e50dd460e9919d2e8fed2b160 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/24997f7e50dd460e9919d2e8fed2b160 2024-12-03T02:36:28,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/740e1d03cb584b39ab32cadc0635003a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/740e1d03cb584b39ab32cadc0635003a 2024-12-03T02:36:28,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ba27fda479e446a692684bc04aca409e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ba27fda479e446a692684bc04aca409e 2024-12-03T02:36:28,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8dc719f62feb4f9fbff6a9d56e787ca6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/94cd0da605164fdca46e09f912e382b2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/ca3a3b35135e4b7392c9e6d6c76e9e7c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c808195946fe41f78a90e08db8ad8b0c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fb692df026d74ccca32fe0cda20e80e0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/a0ef94733d4f4bf7b3068add276fc38b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/83b652794c1e4f91a6479e4db2bfd244, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d085d369080f41d1b5965b09d137cef6, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fec0ed01480b466d94daf236b24d84a5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/e4bb14f751854461b3d746d78dc20416, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/7677581d113d4edcb36f3b1db6ea2459, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/21ea997afa844c3b97ff9878e3798ea5, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8300ace2d13341a39c0441bb0e6a9002, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/731b8cb76b5842e2a05aeae76cd50f73, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/858ae92b8c6e4439be3055efe83958db, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b187bd7ebc5a4e5bb779bb23419a045f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c758b742e2684aadb473de2ce9145812, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/f21c611e3e074c4a9844390a89976f2b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b14051911fb04e3685a77c841ef4036b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2f5b1d863c524ee1970a6110ace2fde3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d4660dfc4e3648eab86728f09ac3d5e3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/484469bc5b3449dd8baba8f517bfc04a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/3c4b40b69fc5462f9eada6933e9d8db4, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2ee0a871ccb34519a3043b19077e38fa, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/40baeb3871114625898dfdbbc3c53fd1, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/1d9d4aed1cab4c06bfa25cb36dfd352b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/289897cf415a48ea8b50ebdf576a4f99, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b8d4553c36a34f69a057930e486ff982, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/4f17fe443c174ffb9af2816f64b51178, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/730052e4c83a461bbb01084a656e3c60] to archive 2024-12-03T02:36:28,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:36:28,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8dc719f62feb4f9fbff6a9d56e787ca6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8dc719f62feb4f9fbff6a9d56e787ca6 2024-12-03T02:36:28,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/94cd0da605164fdca46e09f912e382b2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/94cd0da605164fdca46e09f912e382b2 2024-12-03T02:36:28,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/ca3a3b35135e4b7392c9e6d6c76e9e7c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/ca3a3b35135e4b7392c9e6d6c76e9e7c 2024-12-03T02:36:28,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c808195946fe41f78a90e08db8ad8b0c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c808195946fe41f78a90e08db8ad8b0c 2024-12-03T02:36:28,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fb692df026d74ccca32fe0cda20e80e0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fb692df026d74ccca32fe0cda20e80e0 2024-12-03T02:36:28,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/a0ef94733d4f4bf7b3068add276fc38b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/a0ef94733d4f4bf7b3068add276fc38b 2024-12-03T02:36:28,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/83b652794c1e4f91a6479e4db2bfd244 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/83b652794c1e4f91a6479e4db2bfd244 2024-12-03T02:36:28,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d085d369080f41d1b5965b09d137cef6 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d085d369080f41d1b5965b09d137cef6 2024-12-03T02:36:28,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fec0ed01480b466d94daf236b24d84a5 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/fec0ed01480b466d94daf236b24d84a5 2024-12-03T02:36:28,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/e4bb14f751854461b3d746d78dc20416 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/e4bb14f751854461b3d746d78dc20416 2024-12-03T02:36:28,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/7677581d113d4edcb36f3b1db6ea2459 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/7677581d113d4edcb36f3b1db6ea2459 2024-12-03T02:36:28,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/21ea997afa844c3b97ff9878e3798ea5 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/21ea997afa844c3b97ff9878e3798ea5 2024-12-03T02:36:28,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8300ace2d13341a39c0441bb0e6a9002 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8300ace2d13341a39c0441bb0e6a9002 2024-12-03T02:36:28,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/731b8cb76b5842e2a05aeae76cd50f73 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/731b8cb76b5842e2a05aeae76cd50f73 2024-12-03T02:36:28,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/858ae92b8c6e4439be3055efe83958db to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/858ae92b8c6e4439be3055efe83958db 2024-12-03T02:36:28,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b187bd7ebc5a4e5bb779bb23419a045f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b187bd7ebc5a4e5bb779bb23419a045f 2024-12-03T02:36:28,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c758b742e2684aadb473de2ce9145812 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/c758b742e2684aadb473de2ce9145812 2024-12-03T02:36:28,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/f21c611e3e074c4a9844390a89976f2b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/f21c611e3e074c4a9844390a89976f2b 2024-12-03T02:36:28,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b14051911fb04e3685a77c841ef4036b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b14051911fb04e3685a77c841ef4036b 2024-12-03T02:36:28,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2f5b1d863c524ee1970a6110ace2fde3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2f5b1d863c524ee1970a6110ace2fde3 2024-12-03T02:36:28,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d4660dfc4e3648eab86728f09ac3d5e3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/d4660dfc4e3648eab86728f09ac3d5e3 2024-12-03T02:36:28,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/484469bc5b3449dd8baba8f517bfc04a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/484469bc5b3449dd8baba8f517bfc04a 2024-12-03T02:36:28,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/3c4b40b69fc5462f9eada6933e9d8db4 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/3c4b40b69fc5462f9eada6933e9d8db4 2024-12-03T02:36:28,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2ee0a871ccb34519a3043b19077e38fa to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/2ee0a871ccb34519a3043b19077e38fa 2024-12-03T02:36:28,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/40baeb3871114625898dfdbbc3c53fd1 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/40baeb3871114625898dfdbbc3c53fd1 2024-12-03T02:36:28,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/1d9d4aed1cab4c06bfa25cb36dfd352b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/1d9d4aed1cab4c06bfa25cb36dfd352b 2024-12-03T02:36:28,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/289897cf415a48ea8b50ebdf576a4f99 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/289897cf415a48ea8b50ebdf576a4f99 2024-12-03T02:36:28,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b8d4553c36a34f69a057930e486ff982 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/b8d4553c36a34f69a057930e486ff982 2024-12-03T02:36:28,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/4f17fe443c174ffb9af2816f64b51178 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/4f17fe443c174ffb9af2816f64b51178 2024-12-03T02:36:28,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/730052e4c83a461bbb01084a656e3c60 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/730052e4c83a461bbb01084a656e3c60 2024-12-03T02:36:28,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/64f1f7a3cb094cf0910550c8e188572d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/29b56d7aa4114ba699294a7beeb3101a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1c0d121e45e748a780f9557a3cfc4294, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/a0390e8b559a4dbd89fcb620a2034b47, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3cbc6a436374d57a374b1bf28b81ca3, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/92e49de1315e4b1ab873e6ac8aeeb6e7, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37ca2f3047fd4af0915d2c80f10ceb6b, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1e14f52551f24156990c76708a168c98, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ced6619631dc438192b1b70fa8f17df2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/eb2c26872a294879a0a32f0ae590e849, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/8a3349a364064a66aca924c9802e0b4a, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/352dfb5493f448978ca4e7378c625efa, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/c991c64b964f4db9b0297dcfb095be57, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7ee44e8caf684eac9d473eaa91a88913, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f4b4cc553e241228cdaea47a92bb52e, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/afce4b9f4d4e46b79d4fc99fd896e127, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/fde52fabc5f74749b7326ba35dbfef60, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/76b29038d3884e729fbd74997c161d8f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7f3aa8567f0346f3b63d8daa170d9dd2, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ae33f2d0fe5241728f9598b418cd8784, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/14fc35b74e1840a1a9a1c33505d5be36, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3b4a6d6b69547d196263a724db29d4f, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f7822645ba74505aa46652be90c2a00, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/b316d88586844a26942ee768e3c92eff, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/998eb948c087492781097320da787891, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37f96729af824b9da65f8bafe1738b33, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/663dd2e3d1c04d76b51d19544f2d652d, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/f78005aa231f4f21a4129b732c86d63c, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/20bee25e2ec745e3a773c8e86cf729e0, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/e7ec4836acd34fbcbf9a2e6db865fdd2] to archive 2024-12-03T02:36:28,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T02:36:28,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/64f1f7a3cb094cf0910550c8e188572d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/64f1f7a3cb094cf0910550c8e188572d 2024-12-03T02:36:28,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/29b56d7aa4114ba699294a7beeb3101a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/29b56d7aa4114ba699294a7beeb3101a 2024-12-03T02:36:28,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1c0d121e45e748a780f9557a3cfc4294 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1c0d121e45e748a780f9557a3cfc4294 2024-12-03T02:36:28,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/a0390e8b559a4dbd89fcb620a2034b47 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/a0390e8b559a4dbd89fcb620a2034b47 2024-12-03T02:36:28,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3cbc6a436374d57a374b1bf28b81ca3 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3cbc6a436374d57a374b1bf28b81ca3 2024-12-03T02:36:28,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/92e49de1315e4b1ab873e6ac8aeeb6e7 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/92e49de1315e4b1ab873e6ac8aeeb6e7 2024-12-03T02:36:28,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37ca2f3047fd4af0915d2c80f10ceb6b to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37ca2f3047fd4af0915d2c80f10ceb6b 2024-12-03T02:36:28,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1e14f52551f24156990c76708a168c98 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/1e14f52551f24156990c76708a168c98 2024-12-03T02:36:28,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ced6619631dc438192b1b70fa8f17df2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ced6619631dc438192b1b70fa8f17df2 2024-12-03T02:36:28,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/eb2c26872a294879a0a32f0ae590e849 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/eb2c26872a294879a0a32f0ae590e849 2024-12-03T02:36:28,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/8a3349a364064a66aca924c9802e0b4a to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/8a3349a364064a66aca924c9802e0b4a 2024-12-03T02:36:28,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/352dfb5493f448978ca4e7378c625efa to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/352dfb5493f448978ca4e7378c625efa 2024-12-03T02:36:28,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/c991c64b964f4db9b0297dcfb095be57 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/c991c64b964f4db9b0297dcfb095be57 2024-12-03T02:36:28,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7ee44e8caf684eac9d473eaa91a88913 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7ee44e8caf684eac9d473eaa91a88913 2024-12-03T02:36:28,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f4b4cc553e241228cdaea47a92bb52e to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f4b4cc553e241228cdaea47a92bb52e 2024-12-03T02:36:28,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/afce4b9f4d4e46b79d4fc99fd896e127 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/afce4b9f4d4e46b79d4fc99fd896e127 2024-12-03T02:36:28,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/fde52fabc5f74749b7326ba35dbfef60 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/fde52fabc5f74749b7326ba35dbfef60 2024-12-03T02:36:28,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/76b29038d3884e729fbd74997c161d8f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/76b29038d3884e729fbd74997c161d8f 2024-12-03T02:36:28,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7f3aa8567f0346f3b63d8daa170d9dd2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/7f3aa8567f0346f3b63d8daa170d9dd2 2024-12-03T02:36:28,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ae33f2d0fe5241728f9598b418cd8784 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/ae33f2d0fe5241728f9598b418cd8784 2024-12-03T02:36:28,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/14fc35b74e1840a1a9a1c33505d5be36 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/14fc35b74e1840a1a9a1c33505d5be36 2024-12-03T02:36:28,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3b4a6d6b69547d196263a724db29d4f to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/d3b4a6d6b69547d196263a724db29d4f 2024-12-03T02:36:28,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f7822645ba74505aa46652be90c2a00 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/9f7822645ba74505aa46652be90c2a00 2024-12-03T02:36:28,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/b316d88586844a26942ee768e3c92eff to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/b316d88586844a26942ee768e3c92eff 2024-12-03T02:36:28,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/998eb948c087492781097320da787891 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/998eb948c087492781097320da787891 2024-12-03T02:36:28,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37f96729af824b9da65f8bafe1738b33 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/37f96729af824b9da65f8bafe1738b33 2024-12-03T02:36:28,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/663dd2e3d1c04d76b51d19544f2d652d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/663dd2e3d1c04d76b51d19544f2d652d 2024-12-03T02:36:28,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/f78005aa231f4f21a4129b732c86d63c to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/f78005aa231f4f21a4129b732c86d63c 2024-12-03T02:36:28,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/20bee25e2ec745e3a773c8e86cf729e0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/20bee25e2ec745e3a773c8e86cf729e0 2024-12-03T02:36:28,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/e7ec4836acd34fbcbf9a2e6db865fdd2 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/e7ec4836acd34fbcbf9a2e6db865fdd2 2024-12-03T02:36:28,347 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/recovered.edits/466.seqid, newMaxSeqId=466, maxSeqId=4 2024-12-03T02:36:28,348 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. 2024-12-03T02:36:28,348 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 0be0304c4b25813eaf7af443d2898000: Waiting for close lock at 1733193386938Running coprocessor pre-close hooks at 1733193386938Disabling compacts and flushes for region at 1733193386938Disabling writes for close at 1733193386938Obtaining lock to block concurrent updates at 1733193386938Preparing flush snapshotting stores in 0be0304c4b25813eaf7af443d2898000 at 1733193386938Finished memstore snapshotting TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733193386939 (+1 ms)Flushing stores of TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000. at 1733193386939Flushing 0be0304c4b25813eaf7af443d2898000/A: creating writer at 1733193386939Flushing mob file 0be0304c4b25813eaf7af443d2898000/A: appending metadata at 1733193386943 (+4 ms)Flushing mob file 0be0304c4b25813eaf7af443d2898000/A: closing flushed file at 1733193386943Flushing 0be0304c4b25813eaf7af443d2898000/A: appending metadata at 1733193387350 (+407 ms)Flushing 0be0304c4b25813eaf7af443d2898000/A: closing flushed file at 1733193387350Flushing 0be0304c4b25813eaf7af443d2898000/B: creating writer at 1733193387757 (+407 ms)Flushing 0be0304c4b25813eaf7af443d2898000/B: appending metadata at 1733193387759 (+2 ms)Flushing 0be0304c4b25813eaf7af443d2898000/B: closing flushed file at 1733193387759Flushing 0be0304c4b25813eaf7af443d2898000/C: creating writer at 1733193388166 (+407 ms)Flushing 0be0304c4b25813eaf7af443d2898000/C: appending metadata at 1733193388168 (+2 ms)Flushing 0be0304c4b25813eaf7af443d2898000/C: closing flushed file at 1733193388168Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7159463b: reopening flushed file at 1733193388178 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d60e2af: reopening flushed file at 1733193388182 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@274f9ec1: reopening flushed file at 1733193388187 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0be0304c4b25813eaf7af443d2898000 in 1254ms, sequenceid=463, compaction requested=false at 1733193388192 (+5 ms)Writing region close event to WAL at 1733193388335 (+143 ms)Running coprocessor post-close hooks at 1733193388348 (+13 ms)Closed at 1733193388348 2024-12-03T02:36:28,351 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,351 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=0be0304c4b25813eaf7af443d2898000, regionState=CLOSED 2024-12-03T02:36:28,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 because future has completed 2024-12-03T02:36:28,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-03T02:36:28,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 0be0304c4b25813eaf7af443d2898000, server=64035cbb0b46,38721,1733193202350 in 1.5680 sec 2024-12-03T02:36:28,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-12-03T02:36:28,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0be0304c4b25813eaf7af443d2898000, UNASSIGN in 1.5730 sec 2024-12-03T02:36:28,361 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733193388361"}]},"ts":"1733193388361"} 2024-12-03T02:36:28,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-03T02:36:28,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5770 sec 2024-12-03T02:36:28,364 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T02:36:28,364 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T02:36:28,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5870 sec 2024-12-03T02:36:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T02:36:28,925 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:28,925 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T02:36:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:36:28,927 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:36:28,927 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:36:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-03T02:36:28,930 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,932 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C, FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/recovered.edits] 2024-12-03T02:36:28,934 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3431815651f64edeb510cf00b6b23484 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/3431815651f64edeb510cf00b6b23484 2024-12-03T02:36:28,936 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ef4c24c7ba7047909b79c53fa8f4a0ee to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/A/ef4c24c7ba7047909b79c53fa8f4a0ee 2024-12-03T02:36:28,937 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8935e51dd8a9491895dcd8f112b2fd20 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/8935e51dd8a9491895dcd8f112b2fd20 2024-12-03T02:36:28,938 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/ce36c46d187a44039e2e1e497544bbc8 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/B/ce36c46d187a44039e2e1e497544bbc8 2024-12-03T02:36:28,940 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/8d9e22b7c04e4e64ad7a88aab3d7b3d0 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/8d9e22b7c04e4e64ad7a88aab3d7b3d0 2024-12-03T02:36:28,940 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/b42bf22b1f36492dbfe7ff012519b40d to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/C/b42bf22b1f36492dbfe7ff012519b40d 2024-12-03T02:36:28,942 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/recovered.edits/466.seqid to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000/recovered.edits/466.seqid 2024-12-03T02:36:28,942 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/default/TestAcidGuarantees/0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,942 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T02:36:28,943 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T02:36:28,943 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T02:36:28,946 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203008f14e6ada74f1993c775fa0ee8a558_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203008f14e6ada74f1993c775fa0ee8a558_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,947 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203063d4fbd8b844268a842fc41d15920b2_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203063d4fbd8b844268a842fc41d15920b2_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,947 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b4f7cca6c64422c9de58f83d803b5d7_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b4f7cca6c64422c9de58f83d803b5d7_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,948 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120329ab8341155d4e7a95dd58c2a66eff6d_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120329ab8341155d4e7a95dd58c2a66eff6d_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,949 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203315e4f9f6124451695bc26fe2a3d4d24_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203315e4f9f6124451695bc26fe2a3d4d24_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,949 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034132e6f4c6904942b90e361dab6e4c45_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034132e6f4c6904942b90e361dab6e4c45_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,950 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035d31ccac8b4b49e3a44338ea76137e65_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035d31ccac8b4b49e3a44338ea76137e65_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,951 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203662d0a90f6e34e3ab394f3b411762e19_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203662d0a90f6e34e3ab394f3b411762e19_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,951 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036a04c7f8a02e4a16b6680e8d2819cbef_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036a04c7f8a02e4a16b6680e8d2819cbef_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,952 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389d0d51f981c41858eece441da35e174_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389d0d51f981c41858eece441da35e174_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,953 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389fa18b8891f43c98ebf11a0606f3a85_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120389fa18b8891f43c98ebf11a0606f3a85_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,955 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120393f67103b82846d98dcda6ec493a0286_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120393f67103b82846d98dcda6ec493a0286_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,956 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cde2adb852f4678885b3188b044bf33_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cde2adb852f4678885b3188b044bf33_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,957 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a86d9af9415a473a951763a512b78e56_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a86d9af9415a473a951763a512b78e56_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,958 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b551f0f620454e4583b26446fe13d23e_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b551f0f620454e4583b26446fe13d23e_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,958 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba657e4857b7400588280ffb210588df_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba657e4857b7400588280ffb210588df_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,959 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c972032648e1424fa3b7ed9f82aa194b_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c972032648e1424fa3b7ed9f82aa194b_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,960 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d7a9f4ffde104671a2e035ecc6f19721_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d7a9f4ffde104671a2e035ecc6f19721_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,961 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e437fef14b854e82834b6c4e9c463cd1_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e437fef14b854e82834b6c4e9c463cd1_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,961 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e9c7c20411ec4b55a433e3fb733214c2_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e9c7c20411ec4b55a433e3fb733214c2_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,962 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eebc49c0c532417a8160fdd46dba242f_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eebc49c0c532417a8160fdd46dba242f_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,963 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f05e4a72040a4c1080503440b77fe034_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f05e4a72040a4c1080503440b77fe034_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,964 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f202a44eca0b476fa89b01352a3f7d32_0be0304c4b25813eaf7af443d2898000 to hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f202a44eca0b476fa89b01352a3f7d32_0be0304c4b25813eaf7af443d2898000 2024-12-03T02:36:28,964 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T02:36:28,966 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:36:28,968 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T02:36:28,970 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T02:36:28,970 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:36:28,971 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T02:36:28,971 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733193388971"}]},"ts":"9223372036854775807"} 2024-12-03T02:36:28,973 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T02:36:28,973 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0be0304c4b25813eaf7af443d2898000, NAME => 'TestAcidGuarantees,,1733193355203.0be0304c4b25813eaf7af443d2898000.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T02:36:28,973 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T02:36:28,973 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733193388973"}]},"ts":"9223372036854775807"} 2024-12-03T02:36:28,975 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T02:36:28,976 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T02:36:28,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-12-03T02:36:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-03T02:36:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T02:36:29,035 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T02:36:29,045 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 248), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=745 (was 664) - SystemLoadAverage LEAK? -, ProcessCount=9 (was 9), AvailableMemoryMB=3292 (was 3728) 2024-12-03T02:36:29,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T02:36:29,045 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:36:29,045 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:36:29,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:29,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:29,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:36:29,046 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T02:36:29,046 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1945866401, stopped=false 2024-12-03T02:36:29,046 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=64035cbb0b46,44739,1733193201066 2024-12-03T02:36:29,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T02:36:29,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T02:36:29,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:36:29,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:36:29,054 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T02:36:29,055 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:36:29,055 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:36:29,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:29,055 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '64035cbb0b46,38721,1733193202350' ***** 2024-12-03T02:36:29,055 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T02:36:29,056 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T02:36:29,056 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T02:36:29,056 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:36:29,056 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:36:29,056 INFO [RS:0;64035cbb0b46:38721 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T02:36:29,057 INFO [RS:0;64035cbb0b46:38721 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T02:36:29,057 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(959): stopping server 64035cbb0b46,38721,1733193202350 2024-12-03T02:36:29,057 INFO [RS:0;64035cbb0b46:38721 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T02:36:29,057 INFO [RS:0;64035cbb0b46:38721 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;64035cbb0b46:38721. 2024-12-03T02:36:29,057 DEBUG [RS:0;64035cbb0b46:38721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:36:29,057 DEBUG [RS:0;64035cbb0b46:38721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:29,057 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T02:36:29,057 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T02:36:29,057 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T02:36:29,057 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T02:36:29,058 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T02:36:29,058 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T02:36:29,058 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T02:36:29,060 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T02:36:29,060 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T02:36:29,060 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T02:36:29,060 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T02:36:29,060 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T02:36:29,060 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-03T02:36:29,087 INFO [regionserver/64035cbb0b46:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:36:29,089 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/info/3937bd1a5bfe44ea8a6930432b6f1c74 is 95, key is TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7./info:/1733193232276/DeleteFamily/seqid=0 2024-12-03T02:36:29,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742493_1669 (size=6514) 2024-12-03T02:36:29,097 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/info/3937bd1a5bfe44ea8a6930432b6f1c74 2024-12-03T02:36:29,117 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/ns/b599a8c7a422409cbe19e929109b733b is 93, key is TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7./ns:/1733193232233/DeleteFamily/seqid=0 2024-12-03T02:36:29,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742494_1670 (size=5906) 2024-12-03T02:36:29,121 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/ns/b599a8c7a422409cbe19e929109b733b 2024-12-03T02:36:29,145 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/rep_barrier/782905a513044b3591adf4899c59dcc0 is 102, key is TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7./rep_barrier:/1733193232233/DeleteFamily/seqid=0 2024-12-03T02:36:29,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742495_1671 (size=6025) 2024-12-03T02:36:29,259 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T02:36:29,459 DEBUG [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T02:36:29,559 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/rep_barrier/782905a513044b3591adf4899c59dcc0 2024-12-03T02:36:29,584 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/table/f20d7728ec7a4a979381f771b1e00179 is 96, key is TestAcidGuarantees,,1733193206760.a2199fc0aeeef0203e54ad44ea3ac1a7./table:/1733193232233/DeleteFamily/seqid=0 2024-12-03T02:36:29,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742496_1672 (size=5936) 2024-12-03T02:36:29,596 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/table/f20d7728ec7a4a979381f771b1e00179 2024-12-03T02:36:29,602 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/info/3937bd1a5bfe44ea8a6930432b6f1c74 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/info/3937bd1a5bfe44ea8a6930432b6f1c74 2024-12-03T02:36:29,606 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/info/3937bd1a5bfe44ea8a6930432b6f1c74, entries=12, sequenceid=90, filesize=6.4 K 2024-12-03T02:36:29,608 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/ns/b599a8c7a422409cbe19e929109b733b as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/ns/b599a8c7a422409cbe19e929109b733b 2024-12-03T02:36:29,613 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/ns/b599a8c7a422409cbe19e929109b733b, entries=8, sequenceid=90, filesize=5.8 K 2024-12-03T02:36:29,613 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/rep_barrier/782905a513044b3591adf4899c59dcc0 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/rep_barrier/782905a513044b3591adf4899c59dcc0 2024-12-03T02:36:29,618 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/rep_barrier/782905a513044b3591adf4899c59dcc0, entries=6, sequenceid=90, filesize=5.9 K 2024-12-03T02:36:29,619 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/.tmp/table/f20d7728ec7a4a979381f771b1e00179 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/table/f20d7728ec7a4a979381f771b1e00179 2024-12-03T02:36:29,623 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/table/f20d7728ec7a4a979381f771b1e00179, entries=7, sequenceid=90, filesize=5.8 K 2024-12-03T02:36:29,624 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 564ms, sequenceid=90, compaction requested=false 2024-12-03T02:36:29,634 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-03T02:36:29,635 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T02:36:29,635 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T02:36:29,635 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733193389059Running coprocessor pre-close hooks at 1733193389060 (+1 ms)Disabling compacts and flushes for region at 1733193389060Disabling writes for close at 1733193389060Obtaining lock to block concurrent updates at 1733193389060Preparing flush snapshotting stores in 1588230740 at 1733193389060Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733193389061 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733193389061Flushing 1588230740/info: creating writer at 1733193389062 (+1 ms)Flushing 1588230740/info: appending metadata at 1733193389088 (+26 ms)Flushing 1588230740/info: closing flushed file at 1733193389088Flushing 1588230740/ns: creating writer at 1733193389101 (+13 ms)Flushing 1588230740/ns: appending metadata at 1733193389117 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733193389117Flushing 1588230740/rep_barrier: creating writer at 1733193389125 (+8 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733193389145 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733193389145Flushing 1588230740/table: creating writer at 1733193389563 (+418 ms)Flushing 1588230740/table: appending metadata at 1733193389584 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733193389584Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bb3d35e: reopening flushed file at 1733193389601 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65a603e0: reopening flushed file at 1733193389607 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@156a9e8c: reopening flushed file at 1733193389613 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53ba7fcc: reopening flushed file at 1733193389618 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 564ms, sequenceid=90, compaction requested=false at 1733193389624 (+6 ms)Writing region close event to WAL at 1733193389630 (+6 ms)Running coprocessor post-close hooks at 1733193389635 (+5 ms)Closed at 1733193389635 2024-12-03T02:36:29,635 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T02:36:29,659 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(976): stopping server 64035cbb0b46,38721,1733193202350; all regions closed. 2024-12-03T02:36:29,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741834_1010 (size=25344) 2024-12-03T02:36:29,667 DEBUG [RS:0;64035cbb0b46:38721 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/oldWALs 2024-12-03T02:36:29,667 INFO [RS:0;64035cbb0b46:38721 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 64035cbb0b46%2C38721%2C1733193202350.meta:.meta(num 1733193206183) 2024-12-03T02:36:29,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741832_1008 (size=21519782) 2024-12-03T02:36:29,672 DEBUG [RS:0;64035cbb0b46:38721 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/oldWALs 2024-12-03T02:36:29,673 INFO [RS:0;64035cbb0b46:38721 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 64035cbb0b46%2C38721%2C1733193202350:(num 1733193205051) 2024-12-03T02:36:29,673 DEBUG [RS:0;64035cbb0b46:38721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:36:29,673 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:36:29,673 INFO [RS:0;64035cbb0b46:38721 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T02:36:29,673 INFO [RS:0;64035cbb0b46:38721 {}] hbase.ChoreService(370): Chore service for: regionserver/64035cbb0b46:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T02:36:29,673 INFO [RS:0;64035cbb0b46:38721 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T02:36:29,674 INFO [regionserver/64035cbb0b46:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T02:36:29,675 INFO [RS:0;64035cbb0b46:38721 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38721 2024-12-03T02:36:29,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/64035cbb0b46,38721,1733193202350 2024-12-03T02:36:29,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T02:36:29,696 INFO [RS:0;64035cbb0b46:38721 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T02:36:29,697 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [64035cbb0b46,38721,1733193202350] 2024-12-03T02:36:29,704 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/64035cbb0b46,38721,1733193202350 already deleted, retry=false 2024-12-03T02:36:29,705 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 64035cbb0b46,38721,1733193202350 expired; onlineServers=0 2024-12-03T02:36:29,705 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '64035cbb0b46,44739,1733193201066' ***** 2024-12-03T02:36:29,705 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T02:36:29,705 INFO [M:0;64035cbb0b46:44739 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T02:36:29,705 INFO [M:0;64035cbb0b46:44739 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T02:36:29,705 DEBUG [M:0;64035cbb0b46:44739 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T02:36:29,706 DEBUG [M:0;64035cbb0b46:44739 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T02:36:29,706 INFO [M:0;64035cbb0b46:44739 {}] hbase.ChoreService(370): Chore service for: master/64035cbb0b46:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T02:36:29,706 INFO [M:0;64035cbb0b46:44739 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T02:36:29,706 DEBUG [M:0;64035cbb0b46:44739 {}] master.HMaster(1795): Stopping service threads 2024-12-03T02:36:29,706 INFO [M:0;64035cbb0b46:44739 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T02:36:29,706 INFO [M:0;64035cbb0b46:44739 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T02:36:29,706 ERROR [M:0;64035cbb0b46:44739 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-03T02:36:29,707 INFO [M:0;64035cbb0b46:44739 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T02:36:29,707 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T02:36:29,707 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T02:36:29,707 DEBUG [master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.large.0-1733193204745 {}] cleaner.HFileCleaner(306): Exit Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.large.0-1733193204745,5,FailOnTimeoutGroup] 2024-12-03T02:36:29,708 DEBUG [master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.small.0-1733193204752 {}] cleaner.HFileCleaner(306): Exit Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.small.0-1733193204752,5,FailOnTimeoutGroup] 2024-12-03T02:36:29,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T02:36:29,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:36:29,729 DEBUG [M:0;64035cbb0b46:44739 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-03T02:36:29,729 DEBUG [M:0;64035cbb0b46:44739 {}] master.ActiveMasterManager(353): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-03T02:36:29,730 INFO [M:0;64035cbb0b46:44739 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/.lastflushedseqids 2024-12-03T02:36:29,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742497_1673 (size=72) 2024-12-03T02:36:29,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:36:29,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38721-0x1019a5323e20001, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:36:29,807 INFO [RS:0;64035cbb0b46:38721 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T02:36:29,807 INFO [RS:0;64035cbb0b46:38721 {}] regionserver.HRegionServer(1031): Exiting; stopping=64035cbb0b46,38721,1733193202350; zookeeper connection closed. 2024-12-03T02:36:29,808 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39acba5d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39acba5d 2024-12-03T02:36:29,808 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T02:36:30,142 INFO [M:0;64035cbb0b46:44739 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T02:36:30,142 INFO [M:0;64035cbb0b46:44739 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T02:36:30,143 DEBUG [M:0;64035cbb0b46:44739 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T02:36:30,143 INFO [M:0;64035cbb0b46:44739 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:36:30,143 DEBUG [M:0;64035cbb0b46:44739 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:36:30,143 DEBUG [M:0;64035cbb0b46:44739 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T02:36:30,143 DEBUG [M:0;64035cbb0b46:44739 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:36:30,143 INFO [M:0;64035cbb0b46:44739 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=786.15 KB heapSize=963.78 KB 2024-12-03T02:36:30,163 DEBUG [M:0;64035cbb0b46:44739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f58584c322304a7fb4cf1b0f5d54679a is 82, key is hbase:meta,,1/info:regioninfo/1733193206288/Put/seqid=0 2024-12-03T02:36:30,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742498_1674 (size=5672) 2024-12-03T02:36:30,568 INFO [M:0;64035cbb0b46:44739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2197 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f58584c322304a7fb4cf1b0f5d54679a 2024-12-03T02:36:30,593 DEBUG [M:0;64035cbb0b46:44739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ef60747f25344bf8af99af69fcaa1ee7 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1733193234662/Put/seqid=0 2024-12-03T02:36:30,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742499_1675 (size=45774) 2024-12-03T02:36:30,606 INFO [M:0;64035cbb0b46:44739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785.59 KB at sequenceid=2197 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ef60747f25344bf8af99af69fcaa1ee7 2024-12-03T02:36:30,610 INFO [M:0;64035cbb0b46:44739 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ef60747f25344bf8af99af69fcaa1ee7 2024-12-03T02:36:30,629 DEBUG [M:0;64035cbb0b46:44739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ae2c8e909034aa480d8a04c670e67a2 is 69, key is 64035cbb0b46,38721,1733193202350/rs:state/1733193204817/Put/seqid=0 2024-12-03T02:36:30,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073742500_1676 (size=5156) 2024-12-03T02:36:30,637 INFO [M:0;64035cbb0b46:44739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2197 (bloomFilter=true), to=hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ae2c8e909034aa480d8a04c670e67a2 2024-12-03T02:36:30,642 DEBUG [M:0;64035cbb0b46:44739 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f58584c322304a7fb4cf1b0f5d54679a as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f58584c322304a7fb4cf1b0f5d54679a 2024-12-03T02:36:30,647 INFO [M:0;64035cbb0b46:44739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f58584c322304a7fb4cf1b0f5d54679a, entries=8, sequenceid=2197, filesize=5.5 K 2024-12-03T02:36:30,647 DEBUG [M:0;64035cbb0b46:44739 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ef60747f25344bf8af99af69fcaa1ee7 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ef60747f25344bf8af99af69fcaa1ee7 2024-12-03T02:36:30,652 INFO [M:0;64035cbb0b46:44739 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ef60747f25344bf8af99af69fcaa1ee7 2024-12-03T02:36:30,652 INFO [M:0;64035cbb0b46:44739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ef60747f25344bf8af99af69fcaa1ee7, entries=172, sequenceid=2197, filesize=44.7 K 2024-12-03T02:36:30,652 DEBUG [M:0;64035cbb0b46:44739 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ae2c8e909034aa480d8a04c670e67a2 as hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ae2c8e909034aa480d8a04c670e67a2 2024-12-03T02:36:30,656 INFO [M:0;64035cbb0b46:44739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39485/user/jenkins/test-data/704a1425-5f32-e60d-a578-0595e6468d37/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ae2c8e909034aa480d8a04c670e67a2, entries=1, sequenceid=2197, filesize=5.0 K 2024-12-03T02:36:30,657 INFO [M:0;64035cbb0b46:44739 {}] regionserver.HRegion(3140): Finished flush of dataSize ~786.15 KB/805018, heapSize ~963.48 KB/986608, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 514ms, sequenceid=2197, compaction requested=false 2024-12-03T02:36:30,665 INFO [M:0;64035cbb0b46:44739 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:36:30,665 DEBUG [M:0;64035cbb0b46:44739 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733193390143Disabling compacts and flushes for region at 1733193390143Disabling writes for close at 1733193390143Obtaining lock to block concurrent updates at 1733193390143Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733193390143Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=805018, getHeapSize=986848, getOffHeapSize=0, getCellsCount=2394 at 1733193390143Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733193390144 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733193390144Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733193390163 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733193390163Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733193390571 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733193390592 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733193390592Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733193390611 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733193390629 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733193390629Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18d40596: reopening flushed file at 1733193390641 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@479d7889: reopening flushed file at 1733193390647 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d07e493: reopening flushed file at 1733193390652 (+5 ms)Finished flush of dataSize ~786.15 KB/805018, heapSize ~963.48 KB/986608, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 514ms, sequenceid=2197, compaction requested=false at 1733193390657 (+5 ms)Writing region close event to WAL at 1733193390665 (+8 ms)Closed at 1733193390665 2024-12-03T02:36:30,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45879 is added to blk_1073741830_1006 (size=949030) 2024-12-03T02:36:30,670 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T02:36:30,670 INFO [M:0;64035cbb0b46:44739 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T02:36:30,670 INFO [M:0;64035cbb0b46:44739 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44739 2024-12-03T02:36:30,670 INFO [M:0;64035cbb0b46:44739 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T02:36:30,788 INFO [M:0;64035cbb0b46:44739 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T02:36:30,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:36:30,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44739-0x1019a5323e20000, quorum=127.0.0.1:60541, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:36:30,797 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:36:30,801 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T02:36:30,801 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T02:36:30,802 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T02:36:30,802 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/hadoop.log.dir/,STOPPED} 2024-12-03T02:36:30,805 WARN [BP-660169850-172.17.0.3-1733193195623 heartbeating to localhost/127.0.0.1:39485 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T02:36:30,805 WARN [BP-660169850-172.17.0.3-1733193195623 heartbeating to localhost/127.0.0.1:39485 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-660169850-172.17.0.3-1733193195623 (Datanode Uuid b4b48bdb-1fce-440a-b941-c4b6a491ed21) service to localhost/127.0.0.1:39485 2024-12-03T02:36:30,807 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T02:36:30,807 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T02:36:30,808 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d/data/data1/current/BP-660169850-172.17.0.3-1733193195623 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:36:30,808 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/cluster_ff328890-2806-b34d-4e51-76ef7e7fc23d/data/data2/current/BP-660169850-172.17.0.3-1733193195623 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:36:30,809 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T02:36:30,820 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T02:36:30,823 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T02:36:30,824 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T02:36:30,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T02:36:30,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3a59f309-7c3b-0ee5-f6ff-957d27a9638e/hadoop.log.dir/,STOPPED} 2024-12-03T02:36:30,843 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T02:36:31,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down